Web search and page fetch for the LevelCode agent; drop OpenRouter's unbilled web-search fields

app/controllers/api/levelcode/v1/ai_controller.rb

@@ -534,9 +534,15 @@ def current_request_id
         # payload because this is a transparent proxy — we must not drop fields
         # the editor/upstream rely on.
         def chat_params
-          params.permit!.to_h.except("controller", "action", "ai", "format")
+          params.permit!.to_h.except("controller", "action", "ai", "format", *UNBILLED_FIELDS)
         end
 
+        # Fields that would make OpenRouter do paid work the gateway never bills: its web-search plugin and the
+        # search options of a hosted search model. Metering prices tokens at catalog rates (Levelcode.cost_micros) and
+        # never reads OpenRouter's own cost, so these would run on LevelCode's account for free. The editor's web
+        # search goes through AiWebController, which bills it.
+        UNBILLED_FIELDS = %w[plugins web_search_options].freeze
+
         # Per-request output ceiling (M14). Enforcement is per-request-start, so without a per-request
         # bound a burst of concurrent streams could overshoot the dollar budget before their spend
         # lands. Clamps an over-large client value AND sets a default when omitted. The FREE ceiling

app/controllers/api/levelcode/v1/ai_web_controller.rb

@@ -0,0 +1,102 @@
+# frozen_string_literal: true
+
+module Api
+  module Levelcode
+    module V1
+      # Web access for the editor's agent (levelcode docs/WEB-SEARCH.md):
+      #
+      #   POST /api/levelcode/v1/ai/web/search  { queries: [..1-4], count?, recency? }  → results per query + credits
+      #   POST /api/levelcode/v1/ai/web/fetch   { url }                                  → the page's text
+      #
+      # Search is billed like a model turn: the wallet is checked, the worst case (every query) reserved, the queries
+      # that answered settled and written to the ledger as one row (model "web-search"), and the editor gets the same
+      # `levelcode` credits object the chat stream ends with. Fetch is free, behind the same sign-in, agent scope and
+      # per-minute cap as every gateway call; it exists because a browser tab cannot read another site itself.
+      class AiWebController < BaseController
+        before_action :require_agent_scope!
+
+        def search
+          queries = ::Levelcode::WebSearch.clean_queries(params[:queries])
+          if queries.empty?
+            return render_levelcode_error("invalid_request", "queries must hold 1-#{::Levelcode::WebSearch::MAX_QUERIES} search queries", :unprocessable_entity)
+          end
+
+          provider = ::Levelcode::WebSearch.provider
+          return render_levelcode_error("web_search_unavailable", "Web search is not set up on LevelCode Cloud yet.", :service_unavailable) unless provider
+
+          wallet = ::Levelcode::FreeTier.wallet_for(current_levelcode_user)
+          decision = ::Levelcode::Metering.check!(wallet)
+          # Over budget on a throttle plan keeps the cheap model running; a search costs real money per query, so it stops.
+          return render_no_credits if decision.blocked? || decision.throttle
+
+          per_query = ::Levelcode::WebSearch.cost_micros_per_query
+          res = ::Levelcode::Metering.reserve!(wallet, queries.size * per_query)
+          return render_no_credits unless res.ok
+
+          reservation = res.amount
+          settled = false
+          begin
+            count = params[:count].present? ? params[:count].to_i.clamp(1, ::Levelcode::WebSearch::MAX_COUNT) : ::Levelcode::WebSearch::DEFAULT_COUNT
+            recency = ::Levelcode::WebSearch::RECENCY.include?(params[:recency].to_s) ? params[:recency].to_s : nil
+            results = ::Levelcode::WebSearch.run(queries, provider: provider, count: count, recency: recency)
+            cost = results.count { |r| r[:error].nil? } * per_query
+
+            ::Levelcode::Metering.settle!(wallet, reservation, 0, 0, cost)
+            settled = true
+            if cost.positive?
+              RecordUsageJob.perform_async(current_levelcode_user.id, SecureRandom.uuid, "web-search", provider.name,
+                                           0, 0, 0, cost, wallet.period_end.to_i, Time.current.to_i)
+            end
+            render json: { results: results, levelcode: credits(wallet, cost) }
+          ensure
+            ::Levelcode::Metering.settle!(wallet, reservation, 0, 0, 0) unless settled # an exception releases the hold
+          end
+        end
+
+        def fetch
+          url = params[:url].to_s
+          unless url.length <= 2048 && url.match?(%r{\Ahttps?://}i)
+            return render_levelcode_error("invalid_request", "url must be an http or https address", :unprocessable_entity)
+          end
+
+          page = ::Levelcode::WebFetch.get(url)
+          render json: page
+        rescue ::Levelcode::WebFetch::Blocked => e
+          render_levelcode_error("blocked", e.message, :unprocessable_entity)
+        rescue ::Levelcode::WebFetch::NotText => e
+          render_levelcode_error("not_text", e.message, :unsupported_media_type)
+        rescue Net::OpenTimeout, Net::ReadTimeout
+          render_levelcode_error("timeout", "The page did not answer in time.", :gateway_timeout)
+        rescue SocketError, SystemCallError, OpenSSL::SSL::SSLError, EOFError, IOError, URI::InvalidURIError => e
+          render_levelcode_error("unreachable", "The page could not be reached (#{e.class.name.demodulize}).", :bad_gateway)
+        end
+
+        private
+
+        # Web access is an agent capability: the same scope the gateway asks of a tool-calling turn.
+        def require_agent_scope!
+          scopes = (levelcode_token_claims && levelcode_token_claims["scope"].to_s.split(/\s+/)) || []
+          return if scopes.include?("ai:agent")
+
+          render_levelcode_error("insufficient_scope", "This token cannot use web access (missing ai:agent).", :forbidden)
+        end
+
+        def render_no_credits
+          render_levelcode_error("cap_reached", "No credits left for web search in this period.", :payment_required)
+        end
+
+        # The same retail basis as the chat stream's credits frame (AiController#credits_frame).
+        def credits(wallet, cost_micros)
+          state = ::Levelcode::Metering.tranche_state(wallet)
+          {
+            cost_micros: ::Levelcode.retail_micros(cost_micros.to_i, wallet.plan_key),
+            credits_remaining_micros: ::Levelcode.retail_micros(state[:remaining_micros].to_i, wallet.plan_key)
+          }
+        rescue => e
+          Rails.logger.warn("[Api::Levelcode::V1::AiWebController] credits skipped: #{e.class}")
+          nil
+        end
+      end
+    end
+  end
+end

app/services/levelcode/web_fetch.rb

@@ -0,0 +1,118 @@
+# frozen_string_literal: true
+
+require "net/http"
+require "ipaddr"
+require "resolv"
+
+module Levelcode
+  # Reads a web page for the editor's agent (its `web_fetch` tool, from LevelCode in the browser, where a tab cannot
+  # read another site itself; levelcode docs/WEB-SEARCH.md).
+  #
+  # This server must never become a way into its own network: every address a name resolves to is checked, a name
+  # with any private, loopback, link-local or otherwise local address is refused, and the connection goes to the
+  # address that was checked (Net::HTTP#ipaddr=), so a second DNS answer cannot swap in another. Every redirect is
+  # checked the same way. Only text comes back, at most 2 MB of it.
+  module WebFetch
+    module_function
+
+    class Blocked < StandardError; end
+    class NotText < StandardError; end
+
+    MAX_BYTES = 2 * 1024 * 1024
+    MAX_REDIRECTS = 4
+    OPEN_TIMEOUT = 4
+    READ_TIMEOUT = 8
+    USER_AGENT = "Mozilla/5.0 (compatible; LevelCode/1.4; +https://levelcode.ai)"
+    TEXTUAL = %r{\A(text/|application/(xhtml\+xml|xml|json|ld\+json|rss\+xml|atom\+xml|javascript))}i
+
+    BLOCKED = %w[
+      0.0.0.0/8 10.0.0.0/8 100.64.0.0/10 127.0.0.0/8 169.254.0.0/16 172.16.0.0/12 192.0.0.0/24 192.0.2.0/24
+      192.88.99.0/24 192.168.0.0/16 198.18.0.0/15 198.51.100.0/24 203.0.113.0/24 224.0.0.0/4 240.0.0.0/4
+      ::/128 ::1/128 fc00::/7 fe80::/10 ff00::/8 64:ff9b::/96 2001:db8::/32
+    ].map { |c| IPAddr.new(c) }.freeze
+
+    # True for anything that is not the public internet (and for anything that does not parse as an address).
+    def private_address?(ip)
+      addr = IPAddr.new(ip.to_s.delete_prefix("[").delete_suffix("]").sub(/%.*\z/, ""))
+      addr = addr.native if addr.ipv6? && addr.ipv4_mapped?
+      BLOCKED.any? { |r| r.family == addr.family && r.include?(addr) }
+    rescue IPAddr::Error, ArgumentError
+      true
+    end
+
+    def resolve(host)
+      return [ host ] if host.match?(Resolv::IPv4::Regex) || host.match?(Resolv::IPv6::Regex)
+
+      Resolv.getaddresses(host).uniq
+    end
+
+    # { final_url:, status:, content_type:, body:, truncated: }. Raises Blocked, NotText, or a network error.
+    # `resolver` and `connect` are the seams the specs replace.
+    def get(url, resolver: method(:resolve), connect: method(:request))
+      current = url.to_s
+      (MAX_REDIRECTS + 1).times do
+        uri = URI.parse(current)
+        raise Blocked, "only http and https pages can be opened" unless %w[http https].include?(uri.scheme) && uri.host.present?
+        raise Blocked, "addresses with a user name or password are not opened" if uri.userinfo.present?
+
+        host = uri.host.delete_prefix("[").delete_suffix("]")
+        addresses = resolver.call(host)
+        raise Blocked, "#{host} does not resolve" if addresses.empty?
+        raise Blocked, "#{host} is on a private or local network; only public pages are opened" if addresses.any? { |a| private_address?(a) }
+
+        status, headers, body, truncated = connect.call(uri, addresses.first)
+        if (300..399).cover?(status) && headers["location"].present?
+          current = URI.join(current, headers["location"]).to_s
+          next
+        end
+        type = headers["content-type"].to_s
+        raise NotText, "#{type.split(';').first.presence || 'unknown type'} is not a text page" if type.present? && !type.match?(TEXTUAL)
+
+        return { final_url: current, status: status, content_type: type, body: utf8(body, type), truncated: truncated }
+      end
+      raise Blocked, "too many redirects"
+    end
+
+    class TooLarge < StandardError; end
+
+    # One GET to `ip`, with the name kept for the Host header and TLS. Answers [status, headers, body, truncated].
+    def request(uri, ip)
+      http = Net::HTTP.new(uri.host, uri.port)
+      http.ipaddr = ip
+      http.use_ssl = uri.scheme == "https"
+      http.open_timeout = OPEN_TIMEOUT
+      http.read_timeout = READ_TIMEOUT
+      req = Net::HTTP::Get.new(uri.request_uri)
+      req["User-Agent"] = USER_AGENT
+      req["Accept"] = "text/html,application/xhtml+xml,text/plain;q=0.9,application/json;q=0.8,*/*;q=0.5"
+      status = 0
+      headers = {}
+      body = +""
+      truncated = false
+      begin
+        http.start do
+          http.request(req) do |res|
+            status = res.code.to_i
+            headers = res.to_hash.transform_values { |v| Array(v).join(", ") }
+            next if (300..399).cover?(status)
+
+            res.read_body do |chunk|
+              body << chunk
+              raise TooLarge if body.bytesize > MAX_BYTES
+            end
+          end
+        end
+      rescue TooLarge
+        body = body.byteslice(0, MAX_BYTES)
+        truncated = true
+      end
+      [ status, headers, body, truncated ]
+    end
+
+    def utf8(body, content_type)
+      charset = content_type[/charset\s*=\s*"?([\w.:-]+)/i, 1] || "UTF-8"
+      enc = Encoding.find(charset) rescue Encoding::UTF_8
+      body.to_s.dup.force_encoding(enc).encode(Encoding::UTF_8, invalid: :replace, undef: :replace, replace: "�")
+    end
+  end
+end

app/services/levelcode/web_search.rb

@@ -0,0 +1,104 @@
+# frozen_string_literal: true
+
+require "net/http"
+
+module Levelcode
+  # Web search for the editor's agent (its `web_search` tool; levelcode docs/WEB-SEARCH.md). One request carries
+  # 1-4 queries; they run at the same time against the configured search API and are billed per query that
+  # answered, like a model turn: reserved up front, settled after, one ledger row (Api::Levelcode::V1::AiWebController).
+  #
+  # The provider is configured, not hard-wired: LEVELCODE_WEB_SEARCH_PROVIDER (default "brave") and its key in
+  # BRAVE_SEARCH_API_KEY or credentials[:brave_search_api_key]. No key → `provider` is nil → the endpoint answers 503
+  # and the editor tells the model web search is not available, so nothing breaks before the owner sets one.
+  module WebSearch
+    module_function
+
+    MAX_QUERIES = 4
+    MAX_QUERY_CHARS = 400
+    DEFAULT_COUNT = 6
+    MAX_COUNT = 10
+    RECENCY = %w[day week month year].freeze
+    QUERY_TIMEOUT = 6 # seconds; a query that has not answered by then is reported failed, not waited on
+
+    # What one query costs US, in internal COST micro-$ (the wallet's basis). Brave's Data for AI plan lists $5 per
+    # thousand queries; set LEVELCODE_WEB_SEARCH_COST_MICROS to the provider's real price. Retail is derived from it
+    # by the same margin as tokens (Levelcode.retail_micros).
+    def cost_micros_per_query
+      Integer(ENV.fetch("LEVELCODE_WEB_SEARCH_COST_MICROS", "5000"))
+    end
+
+    def provider
+      name = ENV.fetch("LEVELCODE_WEB_SEARCH_PROVIDER", "brave").to_s
+      case name
+      when "brave"
+        key = ENV["BRAVE_SEARCH_API_KEY"].presence || Rails.application.credentials.dig(:brave_search_api_key).presence
+        key && BraveProvider.new(key)
+      end
+    end
+
+    # The editor's queries, cleaned: trimmed, de-duplicated (any case), at most four of at most 400 characters.
+    def clean_queries(raw)
+      Array(raw).map { |q| q.to_s.squish.first(MAX_QUERY_CHARS) }.reject(&:blank?).uniq(&:downcase).first(MAX_QUERIES)
+    end
+
+    # Every query at once. Answers one entry per query, in order: { query:, results: [...] } or { query:, results: [], error: }.
+    def run(queries, provider:, count:, recency:)
+      threads = queries.map do |q|
+        Thread.new do
+          { query: q, results: provider.search(q, count: count, recency: recency) }
+        rescue => e
+          { query: q, results: [], error: e.message.to_s.first(200) }
+        end
+      end
+      threads.each_with_index.map do |t, i|
+        next t.value if t.join(QUERY_TIMEOUT + 1)
+
+        t.kill
+        { query: queries[i], results: [], error: "timed out after #{QUERY_TIMEOUT} s" }
+      end
+    end
+
+    # Brave Search's web endpoint. `http` is the seam the specs replace: (uri, headers) -> [status, body].
+    class BraveProvider
+      ENDPOINT = "https://api.search.brave.com/res/v1/web/search"
+      FRESHNESS = { "day" => "pd", "week" => "pw", "month" => "pm", "year" => "py" }.freeze
+
+      attr_reader :name
+
+      def initialize(key, http: nil)
+        @key = key
+        @http = http || method(:get)
+        @name = "brave"
+      end
+
+      def search(query, count:, recency:)
+        params = { q: query, count: count, extra_snippets: true, text_decorations: false }
+        params[:freshness] = FRESHNESS[recency] if FRESHNESS[recency]
+        uri = URI(ENDPOINT)
+        uri.query = URI.encode_www_form(params)
+        status, body = @http.call(uri, { "Accept" => "application/json", "X-Subscription-Token" => @key })
+        raise "Brave Search answered HTTP #{status}" unless status == 200
+
+        Array(JSON.parse(body).dig("web", "results")).map do |r|
+          {
+            title: r["title"].to_s,
+            url: r["url"].to_s,
+            snippet: [ r["description"], *Array(r["extra_snippets"]).first(2) ].compact.join(" … "),
+            date: r["page_age"].to_s
+          }
+        end
+      end
+
+      private
+
+      def get(uri, headers)
+        Net::HTTP.start(uri.host, uri.port, use_ssl: true, open_timeout: 3, read_timeout: QUERY_TIMEOUT) do |h|
+          req = Net::HTTP::Get.new(uri)
+          headers.each { |k, v| req[k] = v }
+          res = h.request(req)
+          [ res.code.to_i, res.body.to_s ]
+        end
+      end
+    end
+  end
+end

config/routes.rb

@@ -174,6 +174,9 @@
         # HTML catch-all. `ai/chat` kept as an alias for the SPEC/tests.
         post  "ai/chat/completions", to: "ai#chat"
         post  "ai/chat",             to: "ai#chat"
+        # Web access for the agent's web_search / web_fetch tools (levelcode docs/WEB-SEARCH.md).
+        post  "ai/web/search",       to: "ai_web#search"
+        post  "ai/web/fetch",        to: "ai_web#fetch"
       end
     end
   end

spec/requests/api/levelcode/v1/ai_spec.rb

@@ -612,6 +612,25 @@ def with_scope(scope)
       expect(captured["max_tokens"]).to eq(Levelcode::FREE_MAX_TOKENS)
     end
 
+    it "drops OpenRouter's web-search plugin and search options — paid work metering would never bill" do
+      allow(Levelcode::Metering).to receive(:check!)
+        .and_return(Levelcode::Metering::Decision.new(allowed: true, throttle: false, policy: "stop"))
+      captured = nil
+      allow_any_instance_of(Levelcode::OpenRouterAdapter).to receive(:stream) do |_a, routed, on_chunk:|
+        captured = routed
+        canned_chunks.each { |c| on_chunk.call(c) }
+        Levelcode::OpenRouterAdapter::Result.new(usage: final_usage, model: "moonshotai/kimi-k2.6")
+      end
+
+      post "/api/levelcode/v1/ai/chat",
+           params: { model: "x", stream: true, messages: [ { role: "user", content: "who is it?" } ],
+                     plugins: [ { id: "web", max_results: 10 } ], web_search_options: { search_context_size: "high" } }, as: :json
+
+      expect(response).to have_http_status(:ok)
+      expect(captured.keys).not_to include("plugins", "web_search_options")
+      expect(captured["messages"]).to be_present
+    end
+
     it "402s cap_reached with an upgrade_url when the free monthly cap is hit" do
       allow(Levelcode::Metering).to receive(:check!)
         .and_return(Levelcode::Metering::Decision.new(allowed: false, throttle: false, policy: "stop"))