feat: implement Groq fallback for Gemini failures and update ops docs

add Groq dependency and NAS config entries for fallback provider
implement Gemini -> Groq fallback flow on quota, timeout, and API errors
validate Groq API key with models listing and runtime invocation path
update AI orchestration logic in runner and ai enrichment pipeline
update TODO document to mark Groq auth, integration, and verification as completed

config.nas.yaml

@@ -71,6 +71,12 @@ ai:
   model: gemini-3.6-flash
   max_papers_per_run: 20
 
+  fallback:
+    enabled: true
+    provider: groq
+    api_key_env: GROQ_API_KEY
+    model: llama-3.3-70b-versatile
+
 # NAS mode writes to Joplin through Joplin Terminal, not the desktop Data API.
 joplin:
   enabled: false

paper-monitor-todo.md

@@ -1,6 +1,6 @@
 # Paper Monitor 프로젝트 - 남은 작업 및 검토 TODO
 
-최종 업데이트: 2026-08-21 (IEEE 접근 차단 해제 확인)
+최종 업데이트: 2026-08-22 (Groq API 검증 및 fallback 구현 완료 반영)
 
 이 문서는 지금까지 구축한 Synology NAS 기반 논문 자동 모니터링 시스템에서 **아직 미정이거나 수행하지 못한 항목**, 그리고 **운영 전에 변경 또는 추가 검토가 필요한 항목**을 정리한 TODO 목록이다.
 
@@ -229,31 +229,26 @@ Direct use of automatic function calling (AFC) in Models.generate_content is not
 
 ## 5. Groq fallback
 
-### [ ] Groq API 인증 문제 해결
+### [x] Groq API 인증 문제 해결
-현재 상태:
+완료 상태:
-- Groq API key를 등록했으나 401 `Invalid API Key`
+- Groq API key 인증 성공 확인
-- 새 key 교체 후에도 인증 문제로 일단 보류
+- `models.list()` 및 실제 호출 기반 인증 검증 완료
+- Gemini 실패 시 Groq fallback 연결 및 동작 확인
 
-향후:
+적용 동작:
-- Groq Console에서 project/key 상태 재확인
-- `models.list()` 인증 테스트
-- 인증 성공 후 Gemini fallback provider로 연결
-
-목표:
 ```text
 Gemini 성공
-    -> 완료
+  -> 완료
 
 Gemini quota/timeout/API 오류
-    -> Groq fallback
+  -> Groq fallback
 
 Groq도 실패
-    -> AI 없이 논문 저장
+  -> AI 없이 논문 저장
 ```
 
-### [ ] Groq dependency는 인증 성공 후 정식 추가
+### [x] Groq dependency 정식 추가
-현재 Gemini는 정식 dependency 추가 완료.
+Groq 인증 정상화 이후 dependency 반영 및 실행 검증 완료.
-Groq는 인증 정상화 후 `pyproject.toml`에 추가.
 
 ---
 
@@ -616,7 +611,7 @@ IEEE 네트워크 차단은 해제되었으나 API 승인 상태 및 key 교체
 ## P1 - AI
 - [ ] Gemini 429 RetryInfo/backoff 명시적 처리 (13초 간격이 주 보호, 방어 처리 추가)
 - [ ] Gemini AFC 경고 정리 (현재 동작에 문제 없음, 낮은 우선순위)
-- [ ] Gemini -> Groq fallback (인증 문제 미해결, 파이프라인 차단하지 않음)
+- [x] Gemini -> Groq fallback 구현/검증 완료 (Gemini 실패 시 Groq로 자동 전환)
 
 ## P1 - 소스
 - [ ] Semantic Scholar 연결/접근 안정화 후 재활성화
@@ -672,6 +667,11 @@ scheduler-YYYY-MM-DD_HH-MM-SS.log
 - [x] Gemini 무료 5 RPM 한도에서 429 오류 없음 확인
 - [x] 이전 실패(`ai_status=failed`) Gemini 논문 재시도 구현
 
+## Groq fallback (2026-08-22)
+- [x] Groq API key 인증 성공 확인
+- [x] Groq dependency 정식 반영
+- [x] Gemini 실패 시 Groq fallback 동작 검증 완료
+
 ## IEEE 접근 (2026-08-21)
 - [x] IEEE 네트워크 차단 해제 확인 (`developer.ieee.org` HTTP 200 OK)
 

pyproject.toml

@@ -14,6 +14,7 @@ dependencies = [
   "openai>=1.0",
   "tzdata",
   "google-genai",
+  "groq",
   "pypdf",
 ]
 

src/paper_monitor/ai_enrich.py

@@ -1,6 +1,11 @@
 import json
 import time
 
+
+# ============================================================
+# Gemini rate limiter
+# ============================================================
+
 _GEMINI_MIN_INTERVAL = 13.0
 _GEMINI_LAST_CALL = 0.0
 
@@ -17,10 +22,31 @@ def _wait_for_gemini_rate_limit():
 
     _GEMINI_LAST_CALL = time.monotonic()
 
+
+# ============================================================
+# Common AI prompt
+# ============================================================
+
 SYSTEM = '''You are a technical literature triage assistant for an automotive radar engineer.
 
-Analyze ONLY the supplied metadata.
+Analyze ONLY the supplied metadata or full text.
-Do not invent experimental results, methods, conclusions, or claims that are not supported by the supplied metadata.
+Do not invent experimental results, methods, conclusions, or claims that are not supported by the supplied information.
+
+The primary technical scope is radar engineering, especially:
+- automotive radar
+- FMCW radar
+- radar waveform design
+- radar MIMO
+- radar DOA / angle estimation
+- radar super-resolution
+- radar interference
+- automotive radar antenna
+- imaging radar / 4D radar
+- radar target modeling
+- radar signal processing
+- radar hardware and RF impairments
+
+Generic communications, acoustic DOA, sonar, medical sensing, or non-radar MIMO/antenna research should receive low relevance unless there is a clear technical connection to radar.
 
 Return:
 - relevance: integer from 1 to 5
@@ -69,10 +95,14 @@ AI_RESPONSE_SCHEMA = {
         "categories",
         "summary",
         "reason",
-    ],    
+    ],
 }
 
 
+# ============================================================
+# Common helpers
+# ============================================================
+
 def _make_payload(paper):
     return {
         "title": paper.title,
@@ -84,6 +114,18 @@ def _make_payload(paper):
     }
 
 
+def _make_full_text_payload(paper, full_text):
+    return {
+        "title": paper.title,
+        "abstract": paper.abstract,
+        "venue": paper.venue,
+        "year": paper.year,
+        "full_text": full_text,
+        "current_categories": paper.categories,
+        "current_relevance": paper.relevance,
+    }
+
+
 def _apply_result(paper, data):
     paper.relevance = max(
         1,
@@ -110,6 +152,34 @@ def _apply_result(paper, data):
     return paper
 
 
+def _parse_json_text(text):
+    if not text:
+        raise RuntimeError("AI returned empty response")
+
+    text = text.strip()
+
+    # Defensive handling for models that wrap JSON in Markdown fences.
+    if text.startswith("```"):
+        lines = text.splitlines()
+
+        if lines:
+            lines = lines[1:]
+
+        if lines and lines[-1].strip() == "```":
+            lines = lines[:-1]
+
+        text = "\n".join(lines).strip()
+
+        if text.lower().startswith("json"):
+            text = text[4:].strip()
+
+    return json.loads(text)
+
+
+# ============================================================
+# Gemini - metadata / abstract
+# ============================================================
+
 def enrich_with_gemini(paper, api_key, model):
     if not api_key:
         return paper
@@ -139,13 +209,18 @@ def enrich_with_gemini(paper, api_key, model):
         ),
     )
 
-    data = json.loads(response.text)
+    data = _parse_json_text(response.text)
 
     paper = _apply_result(paper, data)
     paper.ai_analysis_level = "abstract"
 
     return paper
 
+
+# ============================================================
+# Gemini - full text
+# ============================================================
+
 def enrich_with_gemini_full_text(
     paper,
     full_text,
@@ -163,15 +238,10 @@ def enrich_with_gemini_full_text(
 
     client = genai.Client(api_key=api_key)
 
-    payload = {
+    payload = _make_full_text_payload(
-        "title": paper.title,
+        paper,
-        "abstract": paper.abstract,
+        full_text,
-        "venue": paper.venue,
+    )
-        "year": paper.year,
-        "full_text": full_text,
-        "current_categories": paper.categories,
-        "current_relevance": paper.relevance,
-    }
 
     prompt = (
         SYSTEM
@@ -202,14 +272,184 @@ contribution, and relevance to automotive radar when supported by the text.
         ),
     )
 
-    data = json.loads(response.text)
+    data = _parse_json_text(response.text)
 
     paper = _apply_result(paper, data)
     paper.ai_analysis_level = "full_text"
 
     return paper
 
-def enrich_with_openai(paper, api_key, model):
+
+# ============================================================
+# Groq common prompt
+# ============================================================
+
+def _make_groq_prompt(payload, full_text=False):
+    if full_text:
+        instruction = """
+You are given extracted full text from the paper.
+
+Base the analysis primarily on the supplied full text.
+Do not invent information that is not present in the supplied text.
+If the extracted text appears incomplete or corrupted, be conservative.
+
+The summary should describe the paper's main technical approach,
+contribution, and relevance to automotive radar when supported by the text.
+"""
+    else:
+        instruction = """
+Analyze the supplied paper metadata only.
+Do not infer details that are not present in the metadata.
+"""
+
+    output_instruction = """
+Return ONLY one valid JSON object.
+
+The JSON object must have exactly these fields:
+
+{
+  "relevance": 1,
+  "categories": ["Signal Processing"],
+  "summary": "한국어 요약",
+  "reason": "한국어 관련성 설명"
+}
+
+Rules:
+- relevance must be an integer from 1 to 5
+- categories must be a JSON array of strings
+- summary must be written in Korean
+- reason must be written in Korean
+- do not use Markdown code fences
+- do not include text before or after the JSON object
+"""
+
+    return (
+        SYSTEM
+        + instruction
+        + output_instruction
+        + "\n\nPaper data:\n"
+        + json.dumps(payload, ensure_ascii=False)
+    )
+
+
+# ============================================================
+# Groq - metadata / abstract
+# ============================================================
+
+def enrich_with_groq(
+    paper,
+    api_key,
+    model,
+):
+    if not api_key:
+        return paper
+
+    from groq import Groq
+
+    client = Groq(api_key=api_key)
+
+    payload = _make_payload(paper)
+
+    prompt = _make_groq_prompt(
+        payload,
+        full_text=False,
+    )
+
+    response = client.chat.completions.create(
+        model=model,
+        messages=[
+            {
+                "role": "system",
+                "content": SYSTEM,
+            },
+            {
+                "role": "user",
+                "content": prompt,
+            },
+        ],
+        temperature=0.1,
+        response_format={
+            "type": "json_object"
+        },
+    )
+
+    content = response.choices[0].message.content
+
+    data = _parse_json_text(content)
+
+    paper = _apply_result(paper, data)
+    paper.ai_analysis_level = "abstract"
+
+    return paper
+
+
+# ============================================================
+# Groq - full text
+# ============================================================
+
+def enrich_with_groq_full_text(
+    paper,
+    full_text,
+    api_key,
+    model,
+):
+    if not api_key:
+        return paper
+
+    if not full_text or not full_text.strip():
+        return paper
+
+    from groq import Groq
+
+    client = Groq(api_key=api_key)
+
+    payload = _make_full_text_payload(
+        paper,
+        full_text,
+    )
+
+    prompt = _make_groq_prompt(
+        payload,
+        full_text=True,
+    )
+
+    response = client.chat.completions.create(
+        model=model,
+        messages=[
+            {
+                "role": "system",
+                "content": SYSTEM,
+            },
+            {
+                "role": "user",
+                "content": prompt,
+            },
+        ],
+        temperature=0.1,
+        response_format={
+            "type": "json_object"
+        },
+    )
+
+    content = response.choices[0].message.content
+
+    data = _parse_json_text(content)
+
+    paper = _apply_result(paper, data)
+    paper.ai_analysis_level = "full_text"
+
+    return paper
+
+
+# ============================================================
+# OpenAI - legacy / optional
+# ============================================================
+
+def enrich_with_openai(
+    paper,
+    api_key,
+    model,
+):
     if not api_key:
         return paper
 
@@ -224,18 +464,15 @@ def enrich_with_openai(paper, api_key, model):
         instructions=SYSTEM,
         input=json.dumps(
             payload,
-            ensure_ascii=False
+            ensure_ascii=False,
         ),
     )
 
-    text = response.output_text.strip()
+    data = _parse_json_text(
+        response.output_text
+    )
 
-    if text.startswith("```"):
+    return _apply_result(
-        text = text.strip("`")
+        paper,
-
+        data,
-        if text.lower().startswith("json"):
+    )
-            text = text[4:].strip()
-
-    data = json.loads(text)
-
-    return _apply_result(paper, data)

src/paper_monitor/runner.py

@@ -18,6 +18,8 @@ from .joplin import JoplinClient
 from .ai_enrich import (
     enrich_with_gemini,
     enrich_with_gemini_full_text,
+    enrich_with_groq,
+    enrich_with_groq_full_text,
 )
 from .collectors.ieee import (
     IEEECollector,
@@ -351,23 +353,57 @@ def run(config_path, dry_run=False):
         #   A. Papers discovered for the first time in this run
         #   B. Papers from earlier runs whose ai_status == "failed"
         #
+        # Primary provider:
+        #   Gemini
+        #
+        # Fallback provider:
+        #   Groq
+        #
+        # Full-text flow:
+        #   Gemini full-text
+        #       -> Groq full-text
+        #       -> Gemini abstract
+        #       -> Groq abstract
+        #
+        # Abstract-only flow:
+        #   Gemini abstract
+        #       -> Groq abstract
+        #
         # Historical papers with ai_status == "" are NOT automatically
         # processed. This prevents unexpectedly processing the entire DB.
         # ============================================================
 
-
         ai = cfg.get('ai', {})
 
         if ai.get('enabled'):
-            api_key = env_value(
+            gemini_api_key = env_value(
                 ai.get('api_key_env', 'GEMINI_API_KEY')
             )
 
-            model = ai.get(
+            gemini_model = ai.get(
                 'model',
                 'gemini-3.6-flash',
             )
 
+            fallback = ai.get('fallback', {})
+            groq_enabled = bool(
+                fallback.get('enabled', False)
+            ) and fallback.get('provider', 'groq').lower() == 'groq'
+
+            groq_api_key = ''
+            groq_model = fallback.get(
+                'model',
+                'llama-3.3-70b-versatile',
+            )
+
+            if groq_enabled:
+                groq_api_key = env_value(
+                    fallback.get(
+                        'api_key_env',
+                        'GROQ_API_KEY',
+                    )
+                )
+
             max_papers = int(
                 ai.get('max_papers_per_run', 20)
             )
@@ -383,7 +419,9 @@ def run(config_path, dry_run=False):
             candidate_map = {}
 
             for p in new_candidates:
-                candidate_map[paper_key(p.doi, p.title)] = p
+                candidate_map[
+                    paper_key(p.doi, p.title)
+                ] = p
 
             for p in retry_candidates:
                 k = paper_key(p.doi, p.title)
@@ -401,24 +439,28 @@ def run(config_path, dry_run=False):
                 reverse=True,
             )[:max_papers]
 
-            if not api_key:
+            if not gemini_api_key and not groq_api_key:
                 log.warning(
-                    'Gemini API key missing; AI enrichment skipped'
+                    'No usable AI API key configured; '
+                    'AI enrichment skipped'
                 )
 
             else:
                 log.info(
-                    'Gemini enrichment candidates: %d '
+                    'AI enrichment candidates: %d '
-                    '(new=%d retry=%d)',
+                    '(new=%d retry=%d, gemini=%s groq=%s)',
                     len(ai_candidates),
                     len(new_candidates),
                     len(retry_candidates),
+                    'YES' if gemini_api_key else 'NO',
+                    'YES' if groq_api_key else 'NO',
                 )
 
                 for p in ai_candidates:
                     full_text_done = False
                     abstract_done = False
                     had_failure = False
+                    full_text = ''
 
                     # -----------------------------------------
                     # 1. Prefer OA PDF full-text analysis
@@ -440,33 +482,89 @@ def run(config_path, dry_run=False):
                                 p.title,
                             )
 
-                            enrich_with_gemini_full_text(
-                                p,
-                                full_text,
-                                api_key,
-                                model,
-                            )
-
-                            full_text_done = True
-                            p.ai_status = 'done'
-
-                            log.info(
-                                'Gemini full-text enriched: '
-                                'relevance=%d categories=%s title=%s',
-                                p.relevance,
-                                p.categories,
-                                p.title,
-                            )
-
                         except Exception:
                             had_failure = True
 
                             log.exception(
-                                'Full-text AI failed; '
+                                'PDF download/extraction failed; '
                                 'trying abstract fallback: %s',
                                 p.title,
                             )
 
+                        # -------------------------------------
+                        # 1-A. Gemini full-text
+                        # -------------------------------------
+                        if (
+                            full_text
+                            and full_text.strip()
+                            and gemini_api_key
+                        ):
+                            try:
+                                enrich_with_gemini_full_text(
+                                    p,
+                                    full_text,
+                                    gemini_api_key,
+                                    gemini_model,
+                                )
+
+                                full_text_done = True
+                                p.ai_status = 'done'
+
+                                log.info(
+                                    'Gemini full-text enriched: '
+                                    'relevance=%d categories=%s title=%s',
+                                    p.relevance,
+                                    p.categories,
+                                    p.title,
+                                )
+
+                            except Exception:
+                                had_failure = True
+
+                                log.exception(
+                                    'Gemini full-text enrichment failed; '
+                                    'trying Groq full-text fallback: %s',
+                                    p.title,
+                                )
+
+                        # -------------------------------------
+                        # 1-B. Groq full-text fallback
+                        # -------------------------------------
+                        if (
+                            not full_text_done
+                            and full_text
+                            and full_text.strip()
+                            and groq_enabled
+                            and groq_api_key
+                        ):
+                            try:
+                                enrich_with_groq_full_text(
+                                    p,
+                                    full_text,
+                                    groq_api_key,
+                                    groq_model,
+                                )
+
+                                full_text_done = True
+                                p.ai_status = 'done'
+
+                                log.info(
+                                    'Groq full-text enriched: '
+                                    'relevance=%d categories=%s title=%s',
+                                    p.relevance,
+                                    p.categories,
+                                    p.title,
+                                )
+
+                            except Exception:
+                                had_failure = True
+
+                                log.exception(
+                                    'Groq full-text enrichment failed; '
+                                    'trying abstract fallback: %s',
+                                    p.title,
+                                )
+
                     # -----------------------------------------
                     # 2. Abstract fallback
                     # -----------------------------------------
@@ -475,32 +573,71 @@ def run(config_path, dry_run=False):
                         and p.abstract
                         and p.abstract.strip()
                     ):
-                        try:
+                        # -------------------------------------
-                            enrich_with_gemini(
+                        # 2-A. Gemini abstract
-                                p,
+                        # -------------------------------------
-                                api_key,
+                        if gemini_api_key:
-                                model,
+                            try:
-                            )
+                                enrich_with_gemini(
+                                    p,
+                                    gemini_api_key,
+                                    gemini_model,
+                                )
 
-                            abstract_done = True
+                                abstract_done = True
-                            p.ai_status = 'done'
+                                p.ai_status = 'done'
 
-                            log.info(
+                                log.info(
-                                'Gemini abstract enriched: '
+                                    'Gemini abstract enriched: '
-                                'relevance=%d categories=%s title=%s',
+                                    'relevance=%d categories=%s title=%s',
-                                p.relevance,
+                                    p.relevance,
-                                p.categories,
+                                    p.categories,
-                                p.title,
+                                    p.title,
-                            )
+                                )
 
-                        except Exception:
+                            except Exception:
-                            had_failure = True
+                                had_failure = True
 
-                            log.exception(
+                                log.exception(
-                                'Gemini abstract enrichment failed; '
+                                    'Gemini abstract enrichment failed; '
-                                'keeping original metadata: %s',
+                                    'trying Groq abstract fallback: %s',
-                                p.title,
+                                    p.title,
-                            )
+                                )
+
+                        # -------------------------------------
+                        # 2-B. Groq abstract fallback
+                        # -------------------------------------
+                        if (
+                            not abstract_done
+                            and groq_enabled
+                            and groq_api_key
+                        ):
+                            try:
+                                enrich_with_groq(
+                                    p,
+                                    groq_api_key,
+                                    groq_model,
+                                )
+
+                                abstract_done = True
+                                p.ai_status = 'done'
+
+                                log.info(
+                                    'Groq abstract enriched: '
+                                    'relevance=%d categories=%s title=%s',
+                                    p.relevance,
+                                    p.categories,
+                                    p.title,
+                                )
+
+                            except Exception:
+                                had_failure = True
+
+                                log.exception(
+                                    'Groq abstract enrichment failed; '
+                                    'keeping original metadata: %s',
+                                    p.title,
+                                )
 
                     # -----------------------------------------
                     # 3. Final AI status
@@ -525,6 +662,8 @@ def run(config_path, dry_run=False):
 
                     k = paper_key(p.doi, p.title)
 
+                    # Retry candidates are existing DB rows, so persist
+                    # their updated AI status immediately.
                     if k not in new_keys:
                         db.upsert(
                             p,
@@ -589,4 +728,4 @@ def cli():
     p.add_argument('--config', default='config.yaml')
     p.add_argument('--dry-run', action='store_true')
     a = p.parse_args()
-    run(a.config, a.dry_run)
+    run(a.config, a.dry_run)