Upload 303 files

hf_unified_server.py

@@ -35,6 +35,7 @@ import json
 from pathlib import Path
 import httpx
 
+from database import CryptoDatabase
 from ai_models import (
     analyze_chart_points,
     analyze_crypto_sentiment,
@@ -85,6 +86,28 @@ WORKSPACE_ROOT = Path(__file__).parent
 PROVIDERS_CONFIG_PATH = settings.providers_config_path
 FALLBACK_RESOURCE_PATH = WORKSPACE_ROOT / "crypto_resources_unified_2025-11-11.json"
 
+LOG_DIR = WORKSPACE_ROOT / "logs"
+APL_REPORT_PATH = WORKSPACE_ROOT / "PROVIDER_AUTO_DISCOVERY_REPORT.json"
+
+# Ensure log directory exists
+LOG_DIR.mkdir(parents=True, exist_ok=True)
+
+# Initialize SQLite database for real market data persistence
+db = CryptoDatabase()
+
+def tail_log_file(path: Path, max_lines: int = 200) -> List[str]:
+    """Return the last max_lines from a log file, if it exists."""
+    if not path.exists():
+        return []
+    try:
+        with path.open("r", encoding="utf-8", errors="ignore") as f:
+            lines = f.readlines()
+        return lines[-max_lines:]
+    except Exception as e:
+        logger.error(f"Error reading log file {path}: {e}")
+        return []
+
+
 def load_providers_config():
     """Load providers from providers_config_extended.json"""
     try:
@@ -636,6 +659,24 @@ async def get_market():
         if not prices:
             raise HTTPException(status_code=503, detail="Unable to fetch market data")
         
+        # Persist real market data into SQLite database (no mocks)
+        try:
+            for item in prices:
+                payload = {
+                    "symbol": item.get("symbol", "").upper(),
+                    "name": item.get("name"),
+                    "price_usd": item.get("current_price") or item.get("price"),
+                    "volume_24h": item.get("total_volume"),
+                    "market_cap": item.get("market_cap"),
+                    "percent_change_1h": item.get("price_change_1h") or item.get("price_change_percentage_1h_in_currency"),
+                    "percent_change_24h": item.get("price_change_percentage_24h"),
+                    "percent_change_7d": item.get("price_change_percentage_7d_in_currency"),
+                    "rank": item.get("market_cap_rank"),
+                }
+                db.save_price(payload)
+        except Exception as db_err:
+            logger.warning(f"Failed to save market data to DB: {db_err}")
+
         return {
             "total_market_cap": overview.get("total_market_cap", 0),
             "btc_dominance": overview.get("btc_dominance", 0),
@@ -2406,3 +2447,155 @@ async def websocket_endpoint(websocket: WebSocket):
             ws_manager.disconnect(websocket)
         except:
             pass
+
+
+@app.get("/api/market/history")
+async def get_market_history(symbol: str = "BTC", limit: int = 10):
+    """
+    Get historical prices from the real SQLite database.
+
+    This uses only stored real market data inserted by /api/market
+    and never generates synthetic or mock values.
+    """
+    symbol = symbol.upper()
+    try:
+        history = db.get_price_history(symbol, limit=limit)
+    except Exception as e:
+        logger.error(f"Error reading history for {symbol}: {e}")
+        raise HTTPException(status_code=500, detail="Error reading market history")
+
+    if not history:
+        return {
+            "symbol": symbol,
+            "history": [],
+            "count": 0,
+            "message": "No history available",
+        }
+
+    return {
+        "symbol": symbol,
+        "history": history,
+        "count": len(history),
+        "source": "SQLite database (real data)",
+    }
+
+
+@app.get("/api/status")
+async def get_status():
+    """
+    System status endpoint used by the admin UI.
+
+    This reports real-time information about providers and database,
+    without fabricating any market data.
+    """
+    providers_cfg = load_providers_config()
+    providers = providers_cfg or {}
+    validated_count = sum(1 for p in providers.values() if p.get("validated"))
+
+    db_path = Path(db.db_path)
+    db_status = "connected" if db_path.exists() else "initializing"
+
+    return {
+        "system_health": "healthy",
+        "timestamp": datetime.now().isoformat(),
+        "total_providers": len(providers),
+        "validated_providers": validated_count,
+        "database_status": db_status,
+        "apl_available": APL_REPORT_PATH.exists(),
+        "use_mock_data": False,
+    }
+
+
+@app.get("/api/logs/recent")
+async def get_recent_logs():
+    """
+    Return recent log lines for the admin UI.
+
+    We read from the main server log file if available.
+    This does not fabricate content; if there are no logs,
+    an empty list is returned.
+    """
+    log_file = LOG_DIR / "server.log"
+    lines = tail_log_file(log_file, max_lines=200)
+    # Wrap plain text lines as structured entries
+    logs = [{"line": line.rstrip("
+")} for line in lines]
+    return {"logs": logs, "count": len(logs)}
+
+
+@app.get("/api/logs/errors")
+async def get_error_logs():
+    """
+    Return recent error log lines from the same log file.
+
+    This is a best-effort filter based on typical ERROR prefixes.
+    """
+    log_file = LOG_DIR / "server.log"
+    lines = tail_log_file(log_file, max_lines=400)
+    error_lines = [line for line in lines if "ERROR" in line or "WARNING" in line]
+    logs = [{"line": line.rstrip("
+")} for line in error_lines[-200:]]
+    return {"errors": logs, "count": len(logs)}
+
+
+def _load_apl_report() -> Optional[Dict[str, Any]]:
+    """Load the APL (Auto Provider Loader) validation report if available."""
+    if not APL_REPORT_PATH.exists():
+        return None
+    try:
+        with APL_REPORT_PATH.open("r", encoding="utf-8") as f:
+            return json.load(f)
+    except Exception as e:
+        logger.error(f"Error reading APL report: {e}")
+        return None
+
+
+@app.get("/api/apl/summary")
+async def get_apl_summary():
+    """
+    Summary of the Auto Provider Loader (APL) report.
+
+    If the report is missing, we return a clear not_available status
+    instead of fabricating metrics.
+    """
+    report = _load_apl_report()
+    if not report or "stats" not in report:
+        return {
+            "status": "not_available",
+            "message": "APL report not found",
+        }
+
+    stats = report.get("stats", {})
+    return {
+        "status": "ok",
+        "http_candidates": stats.get("total_http_candidates", 0),
+        "http_valid": stats.get("http_valid", 0),
+        "http_invalid": stats.get("http_invalid", 0),
+        "http_conditional": stats.get("http_conditional", 0),
+        "hf_candidates": stats.get("total_hf_candidates", 0),
+        "hf_valid": stats.get("hf_valid", 0),
+        "hf_invalid": stats.get("hf_invalid", 0),
+        "hf_conditional": stats.get("hf_conditional", 0),
+        "timestamp": datetime.now().isoformat(),
+    }
+
+
+@app.get("/api/hf/models")
+async def get_hf_models_from_apl():
+    """
+    Return the list of Hugging Face models discovered by the APL report.
+
+    This is used by the admin UI. The data comes from the real
+    PROVIDER_AUTO_DISCOVERY_REPORT.json file if present.
+    """
+    report = _load_apl_report()
+    if not report:
+        return {"models": [], "count": 0, "source": "none"}
+
+    hf_models = report.get("hf_models", {}).get("results", [])
+    return {
+        "models": hf_models,
+        "count": len(hf_models),
+        "source": "APL report",
+    }
+