libs/luci-lib-httpclient/luasrc/httpclient.lua

   1 -- Copyright 2009 Steven Barth <steven@midlink.org>
   2 -- Licensed to the public under the Apache License 2.0.
   3
   4 require "nixio.util"
   5 local nixio = require "nixio"
   6
   7 local ltn12 = require "luci.ltn12"
   8 local util = require "luci.util"
   9 local table = require "table"
  10 local http = require "luci.http"
  11 local date = require "luci.http.date"
  12 local ip = require "luci.ip"
  13
  14 local type, pairs, ipairs, tonumber, tostring = type, pairs, ipairs, tonumber, tostring
  15 local unpack, string = unpack, string
  16
  17 module "luci.httpclient"
  18
  19 function chunksource(sock, buffer)
  20         buffer = buffer or ""
  21         return function()
  22                 local output
  23                 local _, endp, count = buffer:find("^([0-9a-fA-F]+);?.-\r\n")
  24                 while not count and #buffer <= 1024 do
  25                         local newblock, code = sock:recv(1024 - #buffer)
  26                         if not newblock then
  27                                 return nil, code
  28                         end
  29                         buffer = buffer .. newblock
  30                         _, endp, count = buffer:find("^([0-9a-fA-F]+);?.-\r\n")
  31                 end
  32                 count = tonumber(count, 16)
  33                 if not count then
  34                         return nil, -1, "invalid encoding"
  35                 elseif count == 0 then
  36                         return nil
  37                 elseif count + 2 <= #buffer - endp then
  38                         output = buffer:sub(endp+1, endp+count)
  39                         buffer = buffer:sub(endp+count+3)
  40                         return output
  41                 else
  42                         output = buffer:sub(endp+1, endp+count)
  43                         buffer = ""
  44                         if count - #output > 0 then
  45                                 local remain, code = sock:recvall(count-#output)
  46                                 if not remain then
  47                                         return nil, code
  48                                 end
  49                                 output = output .. remain
  50                                 count, code = sock:recvall(2)
  51                         else
  52                                 count, code = sock:recvall(count+2-#buffer+endp)
  53                         end
  54                         if not count then
  55                                 return nil, code
  56                         end
  57                         return output
  58                 end
  59         end
  60 end
  61
  62
  63 function request_to_buffer(uri, options)
  64         local source, code, msg = request_to_source(uri, options)
  65         local output = {}
  66
  67         if not source then
  68                 return nil, code, msg
  69         end
  70
  71         source, code = ltn12.pump.all(source, (ltn12.sink.table(output)))
  72
  73         if not source then
  74                 return nil, code
  75         end
  76
  77         return table.concat(output)
  78 end
  79
  80 function request_to_source(uri, options)
  81         local status, response, buffer, sock = request_raw(uri, options)
  82         if not status then
  83                 return status, response, buffer
  84         elseif status ~= 200 and status ~= 206 then
  85                 return nil, status, buffer
  86         end
  87
  88         if response.headers["Transfer-Encoding"] == "chunked" then
  89                 return chunksource(sock, buffer)
  90         else
  91                 return ltn12.source.cat(ltn12.source.string(buffer), sock:blocksource())
  92         end
  93 end
  94
  95 function parse_url(uri)
  96         local url, rest, tmp = {}, nil, nil
  97
  98         url.scheme, rest = uri:match("^(%w+)://(.+)$")
  99         if not (url.scheme and rest) then
 100                 return nil
 101         end
 102
 103         url.auth, tmp = rest:match("^([^@]+)@(.+)$")
 104         if url.auth and tmp then
 105                 rest = tmp
 106         end
 107
 108         url.host, tmp = rest:match("^%[([0-9a-fA-F:]+)%](.*)$")
 109         if url.host and tmp then
 110                 url.ip6addr = ip.IPv6(url.host)
 111                 if not url.ip6addr then
 112                         return nil
 113                 end
 114                 url.host = string.format("[%s]", url.ip6addr:string())
 115                 rest = tmp
 116         else
 117                 url.host, tmp = rest:match("^(%d+%.%d+%.%d+%.%d+)(.*)$")
 118                 if url.host and tmp then
 119                         url.ipaddr = ip.IPv4(url.host)
 120                         if not url.ipaddr then
 121                                 return nil
 122                         end
 123                         url.host = url.ipaddr:string()
 124                         rest = tmp
 125                 else
 126                         url.host, tmp = rest:match("^([0-9a-zA-Z%.%-]+)(.*)$")
 127                         if url.host and tmp then
 128                                 rest = tmp
 129                         else
 130                                 return nil
 131                         end
 132                 end
 133         end
 134
 135         url.port, tmp = rest:match("^:(%d+)(.*)$")
 136         if url.port and tmp then
 137                 url.port = tonumber(url.port)
 138                 rest = tmp
 139                 if url.port < 1 or url.port > 65535 then
 140                         return nil
 141                 end
 142         end
 143
 144         if url.scheme == "http" then
 145                 url.port = url.port or 80
 146                 url.default_port = (url.port == 80)
 147         elseif url.scheme == "https" then
 148                 url.port = url.port or 443
 149                 url.default_port = (url.port == 443)
 150         end
 151
 152         if rest == "" then
 153                 url.path = "/"
 154         else
 155                 url.path = rest
 156         end
 157
 158         return url
 159 end
 160
 161 --
 162 -- GET HTTP-resource
 163 --
 164 function request_raw(uri, options)
 165         options = options or {}
 166
 167         if options.params then
 168                 uri = uri .. '?' .. http.urlencode_params(options.params)
 169         end
 170
 171         local url = parse_url(uri)
 172
 173         if not url then
 174                 return nil, -1, "unable to parse URI"
 175         end
 176
 177         if url.scheme ~= "http" and url.scheme ~= "https" then
 178                 return nil, -2, "protocol not supported"
 179         end
 180
 181         options.depth = options.depth or 10
 182         local headers = options.headers or {}
 183         local protocol = options.protocol or "HTTP/1.1"
 184         headers["User-Agent"] = headers["User-Agent"] or "LuCI httpclient 0.1"
 185
 186         if headers.Connection == nil then
 187                 headers.Connection = "close"
 188         end
 189
 190         if url.auth and not headers.Authorization then
 191                 headers.Authorization = "Basic " .. nixio.bin.b64encode(url.auth)
 192         end
 193
 194         local addr = tostring(url.ip6addr or url.ipaddr or url.host)
 195         local sock, code, msg = nixio.connect(addr, url.port)
 196         if not sock then
 197                 return nil, code, msg
 198         end
 199
 200         sock:setsockopt("socket", "sndtimeo", options.sndtimeo or 15)
 201         sock:setsockopt("socket", "rcvtimeo", options.rcvtimeo or 15)
 202
 203         if url.scheme == "https" then
 204                 local tls = options.tls_context or nixio.tls()
 205                 sock = tls:create(sock)
 206                 local stat, code, error = sock:connect()
 207                 if not stat then
 208                         return stat, code, error
 209                 end
 210         end
 211
 212         -- Pre assemble fixes
 213         if protocol == "HTTP/1.1" then
 214                 headers.Host = headers.Host or
 215                         (url.default_port and url.host or string.format("%s:%d", url.host, url.port))
 216         end
 217
 218         if type(options.body) == "table" then
 219                 options.body = http.urlencode_params(options.body)
 220         end
 221
 222         if type(options.body) == "string" then
 223                 headers["Content-Length"] = headers["Content-Length"] or #options.body
 224                 headers["Content-Type"] = headers["Content-Type"] or
 225                         "application/x-www-form-urlencoded"
 226                 options.method = options.method or "POST"
 227         end
 228
 229         if type(options.body) == "function" then
 230                 options.method = options.method or "POST"
 231         end
 232
 233         if options.cookies then
 234                 local cookiedata = {}
 235                 for _, c in ipairs(options.cookies) do
 236                         local cdo = c.flags.domain
 237                         local cpa = c.flags.path
 238                         if   (cdo == url.host or cdo == "."..url.host or url.host:sub(-#cdo) == cdo)
 239                          and (cpa == url.path or cpa == "/" or cpa .. "/" == url.path:sub(#cpa+1))
 240                          and (not c.flags.secure or url.scheme == "https")
 241                         then
 242                                 cookiedata[#cookiedata+1] = c.key .. "=" .. c.value
 243                         end
 244                 end
 245                 if headers["Cookie"] then
 246                         headers["Cookie"] = headers["Cookie"] .. "; " .. table.concat(cookiedata, "; ")
 247                 else
 248                         headers["Cookie"] = table.concat(cookiedata, "; ")
 249                 end
 250         end
 251
 252         -- Assemble message
 253         local message = {(options.method or "GET") .. " " .. url.path .. " " .. protocol}
 254
 255         for k, v in pairs(headers) do
 256                 if type(v) == "string" or type(v) == "number" then
 257                         message[#message+1] = k .. ": " .. v
 258                 elseif type(v) == "table" then
 259                         for i, j in ipairs(v) do
 260                                 message[#message+1] = k .. ": " .. j
 261                         end
 262                 end
 263         end
 264
 265         message[#message+1] = ""
 266         message[#message+1] = ""
 267
 268         -- Send request
 269         sock:sendall(table.concat(message, "\r\n"))
 270
 271         if type(options.body) == "string" then
 272                 sock:sendall(options.body)
 273         elseif type(options.body) == "function" then
 274                 local res = {options.body(sock)}
 275                 if not res[1] then
 276                         sock:close()
 277                         return unpack(res)
 278                 end
 279         end
 280
 281         -- Create source and fetch response
 282         local linesrc = sock:linesource()
 283         local line, code, error = linesrc()
 284
 285         if not line then
 286                 sock:close()
 287                 return nil, code, error
 288         end
 289
 290         local protocol, status, msg = line:match("^([%w./]+) ([0-9]+) (.*)")
 291
 292         if not protocol then
 293                 sock:close()
 294                 return nil, -3, "invalid response magic: " .. line
 295         end
 296
 297         local response = {
 298                 status = line, headers = {}, code = 0, cookies = {}, uri = uri
 299         }
 300
 301         line = linesrc()
 302         while line and line ~= "" do
 303                 local key, val = line:match("^([%w-]+)%s?:%s?(.*)")
 304                 if key and key ~= "Status" then
 305                         if type(response.headers[key]) == "string" then
 306                                 response.headers[key] = {response.headers[key], val}
 307                         elseif type(response.headers[key]) == "table" then
 308                                 response.headers[key][#response.headers[key]+1] = val
 309                         else
 310                                 response.headers[key] = val
 311                         end
 312                 end
 313                 line = linesrc()
 314         end
 315
 316         if not line then
 317                 sock:close()
 318                 return nil, -4, "protocol error"
 319         end
 320
 321         -- Parse cookies
 322         if response.headers["Set-Cookie"] then
 323                 local cookies = response.headers["Set-Cookie"]
 324                 for _, c in ipairs(type(cookies) == "table" and cookies or {cookies}) do
 325                         local cobj = cookie_parse(c)
 326                         cobj.flags.path = cobj.flags.path or url.path:match("(/.*)/?[^/]*")
 327                         if not cobj.flags.domain or cobj.flags.domain == "" then
 328                                 cobj.flags.domain = url.host
 329                                 response.cookies[#response.cookies+1] = cobj
 330                         else
 331                                 local hprt, cprt = {}, {}
 332
 333                                 -- Split hostnames and save them in reverse order
 334                                 for part in url.host:gmatch("[^.]*") do
 335                                         table.insert(hprt, 1, part)
 336                                 end
 337                                 for part in cobj.flags.domain:gmatch("[^.]*") do
 338                                         table.insert(cprt, 1, part)
 339                                 end
 340
 341                                 local valid = true
 342                                 for i, part in ipairs(cprt) do
 343                                         -- If parts are different and no wildcard
 344                                         if hprt[i] ~= part and #part ~= 0 then
 345                                                 valid = false
 346                                                 break
 347                                         -- Wildcard on invalid position
 348                                         elseif hprt[i] ~= part and #part == 0 then
 349                                                 if i ~= #cprt or (#hprt ~= i and #hprt+1 ~= i) then
 350                                                         valid = false
 351                                                         break
 352                                                 end
 353                                         end
 354                                 end
 355                                 -- No TLD cookies
 356                                 if valid and #cprt > 1 and #cprt[2] > 0 then
 357                                         response.cookies[#response.cookies+1] = cobj
 358                                 end
 359                         end
 360                 end
 361         end
 362
 363         -- Follow
 364         response.code = tonumber(status)
 365         if response.code and options.depth > 0 then
 366                 if (response.code == 301 or response.code == 302 or response.code == 307)
 367                  and response.headers.Location then
 368                         local nuri = response.headers.Location or response.headers.location
 369                         if not nuri then
 370                                 return nil, -5, "invalid reference"
 371                         end
 372                         if not nuri:match("^%w+://") then
 373                                 nuri = url.default_port and string.format("%s://%s%s", url.scheme, url.host, nuri)
 374                                         or string.format("%s://%s:%d%s", url.scheme, url.host, url.port, nuri)
 375                         end
 376
 377                         options.depth = options.depth - 1
 378                         if options.headers then
 379                                 options.headers.Host = nil
 380                         end
 381                         sock:close()
 382
 383                         return request_raw(nuri, options)
 384                 end
 385         end
 386
 387         return response.code, response, linesrc(true)..sock:readall(), sock
 388 end
 389
 390 function cookie_parse(cookiestr)
 391         local key, val, flags = cookiestr:match("%s?([^=;]+)=?([^;]*)(.*)")
 392         if not key then
 393                 return nil
 394         end
 395
 396         local cookie = {key = key, value = val, flags = {}}
 397         for fkey, fval in flags:gmatch(";%s?([^=;]+)=?([^;]*)") do
 398                 fkey = fkey:lower()
 399                 if fkey == "expires" then
 400                         fval = date.to_unix(fval:gsub("%-", " "))
 401                 end
 402                 cookie.flags[fkey] = fval
 403         end
 404
 405         return cookie
 406 end
 407
 408 function cookie_create(cookie)
 409         local cookiedata = {cookie.key .. "=" .. cookie.value}
 410
 411         for k, v in pairs(cookie.flags) do
 412                 if k == "expires" then
 413                         v = date.to_http(v):gsub(", (%w+) (%w+) (%w+) ", ", %1-%2-%3 ")
 414                 end
 415                 cookiedata[#cookiedata+1] = k .. ((#v > 0) and ("=" .. v) or "")
 416         end
 417
 418         return table.concat(cookiedata, "; ")
 419 end