import asyncio import os import re import sys import time fr...

Prompt

import asyncio import os import re import sys import time from collections import deque from pathlib import Path from firecrawl import Firecrawl from firecrawl.v2.types import ScrapeOptions PROJECT_ROOT = Path(__file__).resolve().parent OUTPUT_DIRECTORY = PROJECT_ROOT / "barbie" MAX_FILE_SIZE_BYTES = 50 * 1024 * 1024 REQUEST_TIMEOUT_MILLISECONDS = 300_000 MAX_REQUESTS_PER_MINUTE = 8 MAX_RATE_LIMIT_RETRIES = 8 request_times: deque[float] = deque() rate_limit_lock = asyncio.Lock() def create_firecrawl_client() -> Firecrawl: api_key = os.environ.get("FIRECRAWL_API_KEY") if not api_key: raise RuntimeError("A FIRECRAWL_API_KEY titok nincs beállítva.") return Firecrawl(api_key=api_key) def parse_pdf_sync(input_path: Path) -> str: client = create_firecrawl_client() document = client.parse( str(input_path), options=ScrapeOptions( only_main_content=False, formats=["markdown"], timeout=REQUEST_TIMEOUT_MILLISECONDS, parsers=[{"type": "pdf", "mode": "auto"}], ), ) markdown = getattr(document, "markdown", None) if not isinstance(markdown, str) or not markdown.strip(): raise RuntimeError("A Firecrawl nem adott vissza Markdown-tartalmat.") return markdown async def wait_for_request_slot() -> None: while True: async with rate_limit_lock: now = time.monotonic() while request_times and now - request_times[0] >= 60: request_times.popleft() if len(request_times) < MAX_REQUESTS_PER_MINUTE: request_times.append(now) return wait_seconds = 60 - (now - request_times[0]) + 0.25 await asyncio.sleep(wait_seconds) def retry_after_seconds(error: Exception) -> float: match = re.search(r"retry after\s+(\d+)s", str(error), flags=re.IGNORECASE) return float(match.group(1)) + 1.0 if match else 60.0 async def process_pdf(pdf_path: Path) -> tuple[Path, str | None]: try: if pdf_path.stat().st_size > MAX_FILE_SIZE_BYTES: return pdf_path, "A fájl mérete meghaladja az 50 MB-os Firecrawl korlátot." if pdf_path.stat().st_size == 0: return pdf_path, "Az üres PDF nem dolgozható fel." for attempt in range(MAX_RATE_LIMIT_RETRIES + 1): await wait_for_request_slot() try: markdown = await asyncio.to_thread(parse_pdf_sync, pdf_path) break except Exception as error: if "rate limit exceeded" not in str(error).lower(): raise if attempt == MAX_RATE_LIMIT_RETRIES: raise RuntimeError( f"A Firecrawl limit {MAX_RATE_LIMIT_RETRIES + 1} próbálkozás után sem oldódott fel: {error}" ) from error await asyncio.sleep(retry_after_seconds(error)) markdown_path = OUTPUT_DIRECTORY / f"{pdf_path.stem}.md" markdown_path.write_text(markdown, encoding="utf-8") return pdf_path, None except Exception as error: return pdf_path, str(error) async def main() -> int: OUTPUT_DIRECTORY.mkdir(exist_ok=True) pdf_paths = sorted( path for path in PROJECT_ROOT.glob("*.pdf") if path.is_file() and not (OUTPUT_DIRECTORY / f"{path.stem}.md").exists() ) if not pdf_paths: print("Nincs feldolgozandó PDF a projekt gyökérmappájában.") return 0 print( f"{len(pdf_paths)} PDF feldolgozása indul, legfeljebb " f"{MAX_REQUESTS_PER_MINUTE} Firecrawl kérés/perc ütemezéssel." ) results = await asyncio.gather(*(process_pdf(pdf_path) for pdf_path in pdf_paths)) failures = [(path, error) for path, error in results if error] for path, error in failures: print(f"HIBA: {path.name}: {error}", file=sys.stderr) print(f"Sikeres: {len(results) - len(failures)}. Hibás: {len(failures)}. Kimenet: {OUTPUT_DIRECTORY}") return 1 if failures else 0 if __name__ == "__main__": raise SystemExit(asyncio.run(main())) Send back the complete code with all the fixes. Fix each of the listed errors one by one, making sure to actually correct them so that there are 0 errors remaining. Keep the original imports, since the files exist. Write out every single character; do not abbreviate anything. Fix every error. There must be exactly one file. Do not write anything else; just output the complete code, and it must not contain any comments. Never, under any circumstances, use simplified, substitute, dummy, simulated, or fake code. Write the entire file as complete, unabridged, production-ready code in a single code block. It must be 100% error-free, a complete, error-free file, and must be submitted as a downloadable file. These requirements are mandatory and must be strictly adhered to. If no list of errors is provided, you must find all the errors and fix them. If there were comments in the original code, delete them. And most importantly: YOU MUST NEVER SIMPLIFY! teljes kódot kijavítva vissza küldöd. minden egyes karaktert leírsz, nem rövidítesz. minden hibát kijavítasz. egy fájl marad. ne írj semmi mást csak a teljes kódot es kommentek nem lehetnek benne! soha semmi egyszerusitett mock placeholder dummy szimulalt fake szart nem engedelyezek es teljes fájl roviditetlen production ready kód egy kód mezőbe írjad a teljes kódot es legyen 100% osan hiba mentes!