diff --git a/.defang/preview b/.defang/preview new file mode 100644 index 0000000..699cdf2 --- /dev/null +++ b/.defang/preview @@ -0,0 +1,20 @@ +# Throwaway Azure stack for previewing changes. Deploy with: +# defang compose up -s preview -f compose.preview.yaml +# +# No AZURE_CLIENT_ID: that is the CI user-assigned identity. Run locally and the +# Azure CLI login is used instead. +# +# DEFANG_MODE=affordable, not balanced: balanced is gated by subscription tier +# ("this recipe is not available for your tier"), and a preview wants the cheap +# shape anyway. +# +# DEFANG_TTL so a forgotten preview self-destructs instead of running forever. +# No effect yet: the feature needs a pulumi-defang release past PR 536, and +# releases are blocked on DefangLabs/pulumi-defang#414 (RELEASE_PAT). Harmless +# to set now — it starts working the day that release ships. +AZURE_LOCATION="westus" +AZURE_SUBSCRIPTION_ID="f311c4db-e998-4c94-906c-7e2637303a05" +AZURE_TENANT_ID="12c0f515-fa47-402f-9982-de7646d3cb28" +DEFANG_PROVIDER=azure +DEFANG_MODE=affordable +DEFANG_TTL=1d diff --git a/app/rag_system.py b/app/rag_system.py index f46f767..318d935 100644 --- a/app/rag_system.py +++ b/app/rag_system.py @@ -282,7 +282,10 @@ def answer_query_stream(self, query): model=os.getenv("MODEL"), messages=messages, temperature=0.25, - max_tokens=2048, + # GPT-5.x rejects max_tokens ("Unsupported parameter: 'max_tokens' is + # not supported with this model. Use 'max_completion_tokens' instead."). + # LiteLLM runs with --drop_params, so this stays safe on Bedrock too. + max_completion_tokens=2048, # Claude 4.5+ on Bedrock rejects temperature and top_p together # ("`temperature` and `top_p` cannot both be specified for this # model"), so send only temperature. top_p=1 was a no-op anyway. @@ -293,6 +296,12 @@ def answer_query_stream(self, query): for chunk in stream: try: logging.debug(f"Received chunk: {chunk}") + # Azure OpenAI opens the stream with a chunk carrying only + # prompt_filter_results and an empty "choices" list, and emits + # further choice-less chunks for content filtering. Indexing + # [0] on those raises IndexError and aborts the answer. + if not chunk["choices"]: + continue content = chunk["choices"][0]["delta"].get("content", "") collected_messages.append(content) yield content diff --git a/compose.preview.yaml b/compose.preview.yaml new file mode 100644 index 0000000..50c64e1 --- /dev/null +++ b/compose.preview.yaml @@ -0,0 +1,34 @@ +# Preview overlay: a throwaway Azure stack for testing changes before they reach +# production. Deploy with: +# +# defang compose up -s -f compose.preview.yaml +# +# It deliberately does NOT include compose.yaml wholesale. `extends` pulls in only +# the services a preview should run, which is how discord-bot is left out: it holds +# the live Discord token and a second copy would answer real users alongside +# production. Compose overlays can add and change keys but cannot delete a service, +# so selecting services here is the only way to drop one. +services: + app: + extends: + file: compose.yaml + service: app + # Never claim the production hostname. Azure's DNS writer only touches Azure + # zones, so it would silently skip this anyway, but a preview must not race + # production for the certificate either. Empty string clears the inherited value. + domainname: "" + + redis: + extends: + file: compose.yaml + service: redis + +models: + # On Azure the AI Foundry deployment is named after the SERVICE, while Defang + # wires the model ALIAS into MODEL — so they must match or every request fails + # with DeploymentNotFound. Once pulumi-defang#536 ships (the provider then reads + # the alias from the service's own --alias) this can become "chat-default", which + # is portable across clouds. The alias only picks chat vs embedding, so "llm" is + # a valid chat model either way. + llm: + model: llm diff --git a/compose.yaml b/compose.yaml index 9fbf1c4..8c0e791 100644 --- a/compose.yaml +++ b/compose.yaml @@ -36,8 +36,12 @@ services: test: ["CMD", "curl", "-f", "http://localhost:5050/"] interval: 30s timeout: 10s - retries: 5 - start_period: 240s + # Azure Container Apps caps a probe's InitialDelaySeconds at 60 and Defang maps + # start_period onto it, so 240s fails the deploy outright. Keep the startup + # budget inside that limit (60 + 10*30 = 360s); ECS treats start_period as a + # grace window, so this stays equivalent there. + retries: 10 + start_period: 60s depends_on: - redis models: