fix(ai): (FTI-6403) Fixed error with Azure path override formatting

changelog/unreleased/kong/fix-ai-azure-incorrect-path-overriding.yml

@@ -0,0 +1,3 @@
+message: "**ai-proxy**: Fixed a bug where `model.options.upstream_path` overrides always returned 404 with the Azure provider."
+type: bugfix
+scope: Plugin

kong/llm/drivers/azure.lua

@@ -118,10 +118,18 @@ function _M.configure_request(conf)
     parsed_url = socket_url.parse(url)
   end
 
+  -- if the path is read from a URL capture, ensure that it is valid
+  parsed_url.path = string_gsub(parsed_url.path, "^/*", "/")
+
+  -- Azure-specific - prepend the deployment instance path to the
+  --                  user-override path.
+  if conf.model.options and conf.model.options.upstream_path then
+    conf.model.options.upstream_path = parsed_url.path
+  end
+
   ai_shared.override_upstream_url(parsed_url, conf)
 
-  -- if the path is read from a URL capture, 3re that it is valid
-  parsed_url.path = string_gsub(parsed_url.path, "^/*", "/")
+  kong.log.warn("JJ: ", parsed_url.path)
 
   kong.service.request.set_path(parsed_url.path)
   kong.service.request.set_scheme(parsed_url.scheme)
@@ -143,7 +151,8 @@ function _M.configure_request(conf)
 
   local query_table = kong.request.get_query()
 
-  -- technically min supported version
+  -- Read the api-version from the client if not configured; assume compatibility because
+  -- Azure are (at LEAST) not going to break the OpenAI-standard 'messages array' format.
   query_table["api-version"] = kong.request.get_query_arg("api-version")
                             or (conf.model.options and conf.model.options.azure_api_version)
 

kong/llm/drivers/shared.lua

@@ -96,7 +96,7 @@ _M.upstream_url_format = {
   openai        = fmt("%s://api.openai.com:%s", (openai_override and "http") or "https", (openai_override) or "443"),
   anthropic     = "https://api.anthropic.com:443",
   cohere        = "https://api.cohere.com:443",
-  azure         = "https://%s.openai.azure.com:443/openai/deployments/%s",
+  azure         = (openai_override and fmt("http://%%s.openai.azure.com:%s/openai/deployments/%%s", openai_override)) or "https://%s.openai.azure.com:443/openai/deployments/%s",
   gemini        = "https://generativelanguage.googleapis.com",
   gemini_vertex = "https://%s",
   bedrock       = "https://bedrock-runtime.%s.amazonaws.com",

spec/03-plugins/38-ai-proxy/05-azure_integration_spec.lua

@@ -15,8 +15,19 @@ for _, strategy in helpers.all_strategies() do
       -- set up azure mock fixtures
       local fixtures = {
         http_mock = {},
+        dns_mock = helpers.dns_mock.new({
+          mocks_only = true,      -- don't fallback to "real" DNS
+        }),
       }
 
+      fixtures.dns_mock:A {
+        name = "001-kong-t.openai.azure.com",
+        address = "127.0.0.1",
+      }
+
+      -- openai llm driver will always send to this port, if var is set
+      helpers.setenv("OPENAI_TEST_PORT", tostring(MOCK_PORT))
+
       fixtures.http_mock.azure = [[
         server {
             server_name azure;
@@ -129,6 +140,56 @@ for _, strategy in helpers.all_strategies() do
               }
             }
 
+            location = "/openai/deployments/azure-other-instance/other/operation" {
+              content_by_lua_block {
+                local pl_file = require "pl.file"
+                local json = require("cjson.safe")
+
+                local token = ngx.req.get_headers()["api-key"]
+                if token == "azure-key" then
+                  ngx.req.read_body()
+                  local body, err = ngx.req.get_body_data()
+                  body, err = json.decode(body)
+
+                  if err or (body.messages == ngx.null) then
+                    ngx.status = 400
+                    ngx.print(pl_file.read("spec/fixtures/ai-proxy/openai/llm-v1-chat/responses/bad_request.json"))
+                  else
+                    ngx.status = 200
+                    ngx.print(pl_file.read("spec/fixtures/ai-proxy/openai/llm-v1-chat/responses/good.json"))
+                  end
+                else
+                  ngx.status = 401
+                  ngx.print(pl_file.read("spec/fixtures/ai-proxy/openai/llm-v1-chat/responses/unauthorized.json"))
+                end
+              }
+            }
+
+            location = "/override/path/completely" {
+              content_by_lua_block {
+                local pl_file = require "pl.file"
+                local json = require("cjson.safe")
+
+                local token = ngx.req.get_headers()["api-key"]
+                if token == "azure-key" then
+                  ngx.req.read_body()
+                  local body, err = ngx.req.get_body_data()
+                  body, err = json.decode(body)
+
+                  if err or (body.messages == ngx.null) then
+                    ngx.status = 400
+                    ngx.print(pl_file.read("spec/fixtures/ai-proxy/openai/llm-v1-chat/responses/bad_request.json"))
+                  else
+                    ngx.status = 200
+                    ngx.print(pl_file.read("spec/fixtures/ai-proxy/openai/llm-v1-chat/responses/good.json"))
+                  end
+                else
+                  ngx.status = 401
+                  ngx.print(pl_file.read("spec/fixtures/ai-proxy/openai/llm-v1-chat/responses/unauthorized.json"))
+                end
+              }
+            }
+
         }
       ]]
 
@@ -387,6 +448,97 @@ for _, strategy in helpers.all_strategies() do
       }
       --
 
+      -- Override path with unique Azure operations
+      local chat_override_path_from_params = assert(bp.routes:insert {
+        service = empty_service,
+        protocols = { "http" },
+        strip_path = true,
+        paths = { "~/ai/openai/deployments/(?<azure_deployment>[^#?/]+)(?<operation_path>[^#?]+)$" }
+      })
+      bp.plugins:insert {
+        name = PLUGIN_NAME,
+        route = { id = chat_override_path_from_params.id },
+        config = {
+          route_type = "preserve",
+          auth = {
+            header_name = "api-key",
+            header_value = "azure-key",
+          },
+          model = {
+            name = "gpt-3.5-turbo",
+            provider = "azure",
+            options = {
+              max_tokens = 256,
+              temperature = 1.0,
+              azure_instance = "001-kong-t",
+              upstream_path = "$(uri_captures.operation_path)",
+              azure_deployment_id = "$(uri_captures.azure_deployment)",
+            },
+          },
+        },
+      }
+      --
+
+      -- Override path completely
+      local chat_override_path_completely = assert(bp.routes:insert {
+        service = empty_service,
+        protocols = { "http" },
+        strip_path = true,
+        paths = { "~/override/path/completely$" }
+      })
+      bp.plugins:insert {
+        name = PLUGIN_NAME,
+        route = { id = chat_override_path_completely.id },
+        config = {
+          route_type = "preserve",
+          auth = {
+            header_name = "api-key",
+            header_value = "azure-key",
+          },
+          model = {
+            name = "gpt-3.5-turbo",
+            provider = "azure",
+            options = {
+              max_tokens = 256,
+              temperature = 1.0,
+              azure_instance = "001-kong-t",
+              azure_deployment_id = "gpt-3.5-custom",
+            },
+          },
+        },
+      }
+      --
+
+      -- Override path and expect 404
+      local chat_override_path_incorrectly = assert(bp.routes:insert {
+        service = empty_service,
+        protocols = { "http" },
+        strip_path = true,
+        paths = { "~/override/path/incorrectly$" }
+      })
+      bp.plugins:insert {
+        name = PLUGIN_NAME,
+        route = { id = chat_override_path_incorrectly.id },
+        config = {
+          route_type = "preserve",
+          auth = {
+            header_name = "api-key",
+            header_value = "azure-key",
+          },
+          model = {
+            name = "gpt-3.5-turbo",
+            provider = "azure",
+            options = {
+              max_tokens = 256,
+              temperature = 1.0,
+              azure_instance = "001-kong-t",
+              azure_deployment_id = "gpt-3.5-custom",
+            },
+          },
+        },
+      }
+      --
+
 
 
       -- start kong
@@ -647,6 +799,47 @@ for _, strategy in helpers.all_strategies() do
         assert.equals("request body doesn't contain valid prompts", json.error.message)
       end)
     end)
+
+    describe("azure preserve", function()
+      it("override path from path params", function()
+        local r = client:get("/ai/openai/deployments/azure-other-instance/other/operation", {
+          headers = {
+            ["content-type"] = "application/json",
+            ["accept"] = "application/json",
+          },
+          body = pl_file.read("spec/fixtures/ai-proxy/openai/llm-v1-chat/requests/good.json"),
+        })
+
+        -- validate that the request succeeded, response status 200
+        assert.res_status(200 , r)
+      end)
+
+      it("override path completely", function()
+        local r = client:get("/override/path/completely", {
+          headers = {
+            ["content-type"] = "application/json",
+            ["accept"] = "application/json",
+          },
+          body = pl_file.read("spec/fixtures/ai-proxy/openai/llm-v1-chat/requests/good.json"),
+        })
+
+        -- validate that the request succeeded, response status 200
+        assert.res_status(200 , r)
+      end)
+
+      it("override path incorrectly", function()
+        local r = client:get("/override/path/incorrectly", {
+          headers = {
+            ["content-type"] = "application/json",
+            ["accept"] = "application/json",
+          },
+          body = pl_file.read("spec/fixtures/ai-proxy/openai/llm-v1-chat/requests/good.json"),
+        })
+
+        -- expect it to 404 from the backend
+        assert.res_status(404 , r)
+      end)
+    end)
   end)
 
 end