Send back the complete code with all the fixes. Fix each of ...
Prompt
Send back the complete code with all the fixes. Fix each of the listed errors one by one, making sure to actually correct them so that there are 0 errors remaining. Keep the original imports, since the files exist. Write out every single character; do not abbreviate anything. Fix every error. There must be exactly one file. Do not write anything else; just output the complete code, and it must not contain any comments. Never, under any circumstances, use simplified, substitute, dummy, simulated, or fake code. Write the entire file as complete, unabridged, production-ready code in a single code block. It must be 100% error-free, a complete, error-free file, and must be submitted as a downloadable file. These requirements are mandatory and must be strictly adhered to. If no list of errors is provided, you must find all the errors and fix them. If there were comments in the original code, delete them. And most importantly: YOU MUST NEVER SIMPLIFY! import os import subprocess from pathlib import Path import modal MODEL_NAME = "skt/A.X-K2" MODEL_REVISION = "main" HF_TOKEN = os.environ.get("HF_TOKEN", "") VLLM_REPOSITORY = "https://github.com/SKT-AI/vllm.git" VLLM_BRANCH = "axk2-v0.23.0" VLLM_SOURCE_DIR = "/opt/vllm" VLLM_PORT = 8000 VLLM_BASE_URL = f"http://127.0.0.1:{VLLM_PORT}" N_GPUS = 4 GPU_TYPE = "B300" TIMEOUT_SECONDS = 3600 STARTUP_TIMEOUT_SECONDS = 3300 app = modal.App("axk2-vllm-inference") vllm_image = ( modal.Image.from_registry( "nvidia/cuda:13.1.0-devel-ubuntu22.04", add_python="3.12", ) .entrypoint([]) .apt_install( "build-essential", "ca-certificates", "ccache", "curl", "git", "libibverbs-dev", "ninja-build", ) .env( { "CUDA_HOME": "/usr/local/cuda", "HF_HOME": "/huggingface", "HF_HUB_CACHE": "/huggingface/hub", "HF_HUB_ENABLE_HF_TRANSFER": "1", "HF_XET_HIGH_PERFORMANCE": "1", "PYTHONUNBUFFERED": "1", "TORCH_CUDA_ARCH_LIST": "10.0", "TRANSFORMERS_CACHE": "/huggingface/hub", "VLLM_CACHE_ROOT": "/root/.cache/vllm", } ) .run_commands( "python -m pip install --upgrade pip 'setuptools>=77.0.3,<81.0.0' wheel 'cmake>=3.26.1' ninja 'packaging>=24.2' 'setuptools-scm>=8.0' 'setuptools-rust>=1.9.0' jinja2", "python -m pip install --extra-index-url https://download.pytorch.org/whl/cu130 torch==2.11.0", f"git clone --branch {VLLM_BRANCH} {VLLM_REPOSITORY} {VLLM_SOURCE_DIR}", f"cd {VLLM_SOURCE_DIR} && VLLM_USE_PRECOMPILED=1 VLLM_PRECOMPILED_WHEEL_VARIANT=cu130 python -m pip install --no-build-isolation --extra-index-url https://download.pytorch.org/whl/cu130 .", ) .pip_install( "fastapi[standard]", "hf-transfer", "hf-xet", "httpx", "huggingface_hub", "ray", "requests", ) ) hf_volume = modal.Volume.from_name("huggingface-cache", create_if_missing=True) vllm_volume = modal.Volume.from_name("vllm-cache", create_if_missing=True) @app.function( image=vllm_image, gpu=f"{GPU_TYPE}:{N_GPUS}", volumes={ "/huggingface": hf_volume, "/root/.cache": vllm_volume, }, secrets=[modal.Secret.from_name("huggingface-secret")], timeout=TIMEOUT_SECONDS, min_containers=1, ) @modal.asgi_app() def serve_vllm(): import asyncio import time from contextlib import asynccontextmanager import httpx import requests from fastapi import FastAPI, HTTPException from starlette.background import BackgroundTask from starlette.requests import Request from starlette.responses import StreamingResponse Path("/huggingface/hub").mkdir(parents=True, exist_ok=True) Path("/root/.cache/vllm").mkdir(parents=True, exist_ok=True) process_environment = os.environ.copy() if HF_TOKEN and not process_environment.get("HF_TOKEN"): process_environment["HF_TOKEN"] = HF_TOKEN command = [ "vllm", "serve", MODEL_NAME, "--revision", MODEL_REVISION, "--served-model-name", MODEL_NAME, "--tensor-parallel-size", str(N_GPUS), "--enable-auto-tool-choice", "--tool-call-parser", "hermes", "--reasoning-parser", "deepseek_v3", "--max-model-len", "262144", "--host", "127.0.0.1", "--port", str(VLLM_PORT), "--uvicorn-log-level", "info", ] process = subprocess.Popen( command, env=process_environment, start_new_session=True, ) def stop_process(): if process.poll() is not None: return process.terminate() try: process.wait(timeout=30) except subprocess.TimeoutExpired: process.kill() process.wait(timeout=30) def wait_for_server(): deadline = time.monotonic() + STARTUP_TIMEOUT_SECONDS with requests.Session() as session: while time.monotonic() < deadline: return_code = process.poll() if return_code is not None: raise RuntimeError( f"vLLM exited before becoming healthy with return code {return_code}." ) try: response = session.get( f"{VLLM_BASE_URL}/health", timeout=10, ) if response.status_code == 200: return except requests.RequestException: pass time.sleep(5) stop_process() raise TimeoutError( f"vLLM did not become healthy within {STARTUP_TIMEOUT_SECONDS} seconds." ) try: wait_for_server() except BaseException: stop_process() raise hop_by_hop_headers = { b"connection", b"keep-alive", b"proxy-authenticate", b"proxy-authorization", b"te", b"trailer", b"transfer-encoding", b"upgrade", } @asynccontextmanager async def lifespan(application: FastAPI): limits = httpx.Limits( max_connections=200, max_keepalive_connections=100, keepalive_expiry=30.0, ) timeout = httpx.Timeout( connect=30.0, read=None, write=None, pool=30.0, ) application.state.client = httpx.AsyncClient( base_url=VLLM_BASE_URL, limits=limits, timeout=timeout, ) try: yield finally: await application.state.client.aclose() await asyncio.to_thread(stop_process) fastapi_app = FastAPI(lifespan=lifespan) @fastapi_app.api_route( "/{path:path}", methods=["GET", "POST", "PUT", "DELETE", "OPTIONS", "HEAD", "PATCH"], ) async def proxy(request: Request, path: str): del path client = request.app.state.client target_url = httpx.URL( path=request.url.path, query=request.url.query.encode("utf-8"), ) request_headers = [ (name, value) for name, value in request.headers.raw if name.lower() not in hop_by_hop_headers and name.lower() != b"host" ] proxy_request = client.build_request( method=request.method, url=target_url, headers=request_headers, content=request.stream(), ) try: response = await client.send(proxy_request, stream=True) except httpx.RequestError as error: raise HTTPException( status_code=502, detail=f"The vLLM backend request failed: {error}", ) from error response_headers = { name.decode("latin-1"): value.decode("latin-1") for name, value in response.headers.raw if name.lower() not in hop_by_hop_headers } return StreamingResponse( response.aiter_raw(), status_code=response.status_code, headers=response_headers, background=BackgroundTask(response.close), ) return fastapi_app @app.function( gpu=f"{GPU_TYPE}:{N_GPUS}", volumes={ "/huggingface": hf_volume, "/root/.cache": vllm_volume, }, secrets=[modal.Secret.from_name("huggingface-secret")], timeout=10, ) def gpu_hack(): import threading import time def run_orphaned_gpu_worker(): os.umask(0) os.chdir("/") command = [ "vllm", "serve", MODEL_NAME, "--revision", MODEL_REVISION, "--served-model-name", MODEL_NAME, "--tensor-parallel-size", str(N_GPUS), "--enable-auto-tool-choice", "--tool-call-parser", "hermes", "--reasoning-parser", "deepseek_v3", "--max-model-len", "262144", "--host", "127.0.0.1", "--port", str(VLLM_PORT), "--uvicorn-log-level", "info", ] subprocess.Popen( command, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, start_new_session=True, ) return thread = threading.Thread(target=run_orphaned_gpu_worker) thread.start() time.sleep(2) return "Hack activated. GPU is now running in the background." @app.function( gpu=f"{GPU_TYPE}:{N_GPUS}", volumes={ "/huggingface": hf_volume, "/root/.cache": vllm_volume, }, timeout=3600, ) def query_orphaned_gpu(prompt: str): import httpx with httpx.Client(base_url=VLLM_BASE_URL) as client: response = client.post( "/generate", json={"prompt": prompt, "max_tokens": 512}, ) return response.json()
