Convert the attached technical description, file, or Repomix...

Prompt

Convert the attached technical description, file, or Repomix export into a build prompt. Return only that prompt, not the implementation. Read the entire input character by character, line by line, from beginning to end before writing. Carry every nonexcluded detail into the prompt without omissions or abbreviated behavior. The resulting prompt must: Direct another language model to build the complete system from scratch, using only the prompt. The model will not see the source on which you are now basing your writing. Fully specify all functionality, architecture. Define everything in the prompt, never referring to the original description or codes. Identify every necessary file and its required contents and behavior. Prohibit the use of any of the following: authentication/authorization, access control, security policies/auditing/monitoring/ rate/usage/product limits, quotas, billing/payments, tests/documentation, source/config comments, placeholder/mock/dummy implementations, fake services, TODO, unimplemented methods, omitted files/functions. Never mention the original software’s name when describing a reimplementation. Replace name-bearing identifiers with neutral descriptive identifiers """zai-org/GLM-5.3-Flash on 4xB300 with SGLang. Serving metadata: engine: sglang base_model_repo_id: zai-org/GLM-5.3-Flash base_model_revision: 3f1971b7b5f7a528c9c4ef6212c8785298a8c24a model_family: glm Deployed with MODAL_IMAGE_BUILDER_VERSION=2025.06""" import modal MINUTES = 60 DEFAULT_PORT = 8000 HF_IMAGE_ENV = { "HF_XET_HIGH_PERFORMANCE": "1", } MODEL_PATH = "/flash-endpoint-model/huggingface/hub/models--dealignai--GLM-5.3-Flash-UNCENSORED-FP8/snapshots/d5822b068da8ddb301665b48c58718e6e18a2c3a" SERVED_MODEL_NAME = "bibi" ROUTING_REGION = "us-west" REQUIRE_AUTHENTICATION = False SGLANG_IMAGE_TAG = "lmsysorg/sglang:v0.5.22" AUTOINFERENCE_UTILS_VERSION = "0.2.6" GPU_TYPE = "B300" N_GPUS = 4 GPU = f"{GPU_TYPE}:{N_GPUS}" CPU = 16 MEMORY_MB = 262144 SCALEDOWN_WINDOW = 20 * MINUTES TARGET_INPUTS = 16 STARTUP_TIMEOUT = 60 * MINUTES EXTRA_IMAGE_ENV = { "HF_HUB_ENABLE_HF_TRANSFER": "1", "PYTORCH_CUDA_ALLOC_CONF": "expandable_segments:True,garbage_collection_threshold:0.8", "SGLANG_CACHE_DIR": "/root/jit-cache/sglang", "SGLANG_SSE_KEEPALIVE_INTERVAL": "10", "SGLANG_TIMEOUT_KEEP_ALIVE": "300", } serving_image = ( modal.Image.from_registry(SGLANG_IMAGE_TAG) .run_commands('mkdir -p /root/.cache/deep_gemm /root/jit-cache /opt/autoinf-patches') .run_commands('cat > /opt/autoinf-patches/patch_kpool_clamp.py <<\'AUTOINF_PATCH_EOF\'\n"""Bounds-guard the kpool top-k transform gather (kpool_topk_transform.cuh).\n\nRuns at image build (see serve.py / serve_fp8_kv.py). In the K-selected\nbranch, kpool_topk_transform_kernel trusts fast_topk\'s shared-memory output\nabsolutely:\n\n const auto group_id = s_indices[group_rank];\n ...\n dst[col] = transform_kpool_token(raw_token, ...); // page_table_entry[raw_token]\n\nA degenerate row puts garbage in s_indices and the gather either reads a\nwrong-but-mapped page (silent attention corruption) or faults with Warp\nIllegal Address, killing the whole engine. Coredump-fingerprinted on prod:\n2026-08-28 23:36 UTC replica death, all four dumps in\nkpool_topk_transform_kernel<512> (3x Warp Illegal Address, 1x Warp\nOut-of-range Address) -> the 23:00-24:00 UTC 504 burst. Same class as the\nglm-kpool-repro findings (s10 corrupt / s11 crash).\n\nThe guard rejects group ids outside [0, length) and writes -1, the sentinel\nthis kernel already pads out-of-range columns with. The sequential\nlength<=K branch never reads s_indices and needs no guard.\n\nThe kernel is runtime-compiled from this header (no prebuilt .so carries\nthe symbol), so the edit takes effect at next boot\'s JIT compile. The\nanchored block must appear exactly once; any other count means the image\nlayout drifted and the build fails.\n"""\n\nimport pathlib\nimport sys\n\nTARGET = pathlib.Path(\n "/sgl-workspace/sglang/python/sglang/kernels/jit/csrc/dsa/kpool_topk_transform.cuh"\n)\n\nSTOCK_BLOCK = """\\\n const auto group_rank = col / pool_size;\n const auto group_id = s_indices[group_rank];\n const auto slot = col % pool_size;\n const auto raw_token = group_id * pool_size + slot;\n dst[col] = transform_kpool_token(raw_token, page_table_entry, topk_indices_offset, offset);\n"""\n\nPATCHED_BLOCK = """\\\n const auto group_rank = col / pool_size;\n const auto group_id = s_indices[group_rank];\n const auto slot = col % pool_size;\n const auto raw_token = group_id * pool_size + slot;\n dst[col] = (group_id >= 0 && group_id < length)\n ? transform_kpool_token(raw_token, page_table_entry, topk_indices_offset, offset)\n : -1;\n"""\n\n\ndef main() -> None:\n source = TARGET.read_text()\n if PATCHED_BLOCK in source:\n print("patch_kpool_clamp: already applied")\n return\n count = source.count(STOCK_BLOCK)\n if count != 1:\n sys.exit(\n f"patch_kpool_clamp: expected the s_indices gather block exactly once, "\n f"found {count} - image layout drifted; refusing to build with an "\n f"unguarded kpool top-k gather"\n )\n TARGET.write_text(source.replace(STOCK_BLOCK, PATCHED_BLOCK))\n assert TARGET.read_text().count(PATCHED_BLOCK) == 1\n print("patch_kpool_clamp: guarded kpool_topk_transform_kernel")\n\n\nif __name__ == "__main__":\n main()\nAUTOINF_PATCH_EOF') .run_commands('python /opt/autoinf-patches/patch_kpool_clamp.py') .uv_pip_install( f"autoinference-utils=={AUTOINFERENCE_UTILS_VERSION}", ) .env(HF_IMAGE_ENV | EXTRA_IMAGE_ENV) ) EXTRA_SERVER_ARGS = { "--context-length": "1048576", "--cuda-graph-max-bs-decode": "32", "--disable-shared-experts-fusion": "", "--dist-timeout": "3600", "--dsa-decode-backend": "trtllm", "--dsa-prefill-backend": "trtllm", "--enable-cache-report": "", "--ep-size": "4", "--kv-cache-dtype": "fp8_e4m3", "--max-mamba-cache-size": "1024", "--mem-fraction-static": "0.78", "--moe-runner-backend": "deep_gemm", "--reasoning-parser": "glm45", "--speculative-algorithm": "NEXTN", "--speculative-eagle-topk": "1", "--speculative-num-draft-tokens": "6", "--speculative-num-steps": "5", "--tool-call-parser": "glm47", } SERVER_ARGS = { "--served-model-name": SERVED_MODEL_NAME, } | EXTRA_SERVER_ARGS WARMUP_PAYLOAD = { "model": SERVED_MODEL_NAME, "messages": [{"role": "user", "content": "Reply with JSON facts about Tokyo."}], "max_tokens": 64, "temperature": 0, "response_format": { "type": "json_schema", "json_schema": { "name": "city_facts", "schema": { "type": "object", "properties": { "city": {"type": "string"}, "population": {"type": "integer"}, }, "required": ["city", "population"], "additionalProperties": False, }, "strict": True, }, }, } app = modal.App(name="ep-bibi") @app.server( image=serving_image, gpu=GPU, cpu=CPU, memory=MEMORY_MB, min_containers=0, scaledown_window=SCALEDOWN_WINDOW, port=DEFAULT_PORT, routing_region=ROUTING_REGION, unauthenticated=not REQUIRE_AUTHENTICATION, exit_grace_period=25, startup_timeout=STARTUP_TIMEOUT, target_concurrency=TARGET_INPUTS, volumes={"/flash-endpoint-model": modal.Volume.from_name("endpoint-ep-IldkVgiTmWAWTTZGAdWbCg")}, ) class Server: @modal.enter() def startup(self): from autoinference_utils.endpoint import SGLangEndpoint, warmup_chat_completions self.endpoint = SGLangEndpoint( model_path=MODEL_PATH, worker_port=DEFAULT_PORT, tp=N_GPUS, extra_server_args=SERVER_ARGS, health_timeout=STARTUP_TIMEOUT, health_poll_interval=5.0, ) self.endpoint.start() warmup_chat_completions( port=DEFAULT_PORT, payload=WARMUP_PAYLOAD, successful_requests=2, request_timeout=60.0, ) print(f"{SERVED_MODEL_NAME} ({GPU}) sglang deployment is ready.") @modal.exit() def stop(self): if hasattr(self, "endpoint"): self.endpoint.stop()