Skip to content

Commit

Permalink
bundling mozilla root CA store for when there are no default certs
Browse files Browse the repository at this point in the history 
not all machines have system certs such as in busybox containers
and so by bundling certs from Mozilla we can fallback to them
when system certs are missing
  • Loading branch information
@miki725
miki725 committed Feb 11, 2024
1 parent b4d6e30 commit ebf0596
Show file tree
Hide file tree
Showing 3 changed files with 158 additions and 61 deletions.
5 changes: 3 additions & 2 deletions nimutils/awsclient.nim
View file
Original file line number Diff line number Diff line change
Expand Up @@ -97,8 +97,9 @@ proc newAwsClient*(creds: AwsCredentials, region,
service: string): AwsClient =
let
# TODO - use some kind of template and compile-time variable to put the correct kernel used to build the sdk in the UA?
httpclient = newHttpClient("nimaws-sdk/0.3.3; "&defUserAgent.replace(" ",
"-").toLower&"; darwin/16.7.0")
httpclient = createHttpClient(
userAgent = "nimaws-sdk/0.3.3; "&defUserAgent.replace(" ", "-").toLower&"; darwin/16.7.0",
)
scope = AwsScope(date: getAmzDateString(), region: region, service: service)

return AwsClient(httpClient: httpclient, credentials: creds, scope: scope,
Expand Down
114 changes: 112 additions & 2 deletions nimutils/net.nim
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,43 @@
import std/[asyncfutures, net, httpclient, uri, math, os]
import std/[asyncfutures, net, httpclient, uri, math, os, streams, strutils]
import openssl
import ./managedtmp

template getRootCAStoreContent(): string =
const
caWiki = "https://wiki.mozilla.org/CA/Included_Certificates"
caURL = "https://ccadb.my.salesforce-sites.com/mozilla/IncludedRootsPEMTxt?TrustBitsInclude=Websites"
curlCmd = "curl -fsS --retry 5 " & caURL
(contents, curlExitCode) = gorgeEx(curlCmd, cache="mozilla-root-store")
if curlExitCode != 0:
raise newException(
ValueError,
"Could not downlaod CA root store: " & contents
)
const
opensslCmd = "openssl storeutl -noout -certs /dev/stdin"
(check, checkExitCode) = gorgeEx(opensslCmd, input=contents)
checkLines = check.splitLines()
if checkExitCode != 0:
raise newException(
ValueError,
"Could not validate CA root store certificates. " &
"Maybe server didnt return valid PEM file? " &
check
)
echo("Embedding Mozilla Root CA store with certificates " & checkLines[^1].toLower())
echo("For more information see " & caWiki)
contents

var tmpCAStore = ""
proc getCAStorePath(): string =
const contents = getRootCAStoreContent()
if tmpCAStore != "":
return tmpCAStore
let (stream, tmp) = getNewTempFile("cabundle", ".pem")
stream.write(contents)
stream.close()
tmpCAStore = tmp
return tmp

{.emit: """
#include <stdlib.h>
Expand All @@ -9,7 +48,6 @@ import std/[asyncfutures, net, httpclient, uri, math, os]
#include <stdio.h>
// Cloudflare DNS
const char * dummy_dst = "1.1.1.1";
const int dummy_port = 53;
Expand Down Expand Up @@ -118,3 +156,75 @@ proc safeRequest*(client: HttpClient,
withRetry(retries, firstRetryDelayMs):
return client.request(url = url, httpMethod = httpMethod, body = body,
headers = headers, multipart = multipart)

# https://github.com/nim-lang/Nim/blob/a45f43da3407dbbf8ecd15ce8ecb361af677add7/lib/pure/httpclient.nim#L380-L386
# similar to stdlib but defaults to bundled CAs
proc getSSLContext(caFile: string = ""): SslContext =
if caFile != "":
# note when caFile is provided there is no try..except
# otherwise we would silently fail to bundled CA root store
# if caFile is invalid/does not exist
return newContext(verifyMode = CVerifyPeer, caFile = caFile)
else:
try:
return newContext(verifyMode = CVerifyPeer)
except:
return newContext(verifyMode = CVerifyPeer, caFile = getCAStorePath())

proc createHttpClient*(uri: Uri = parseUri(""),
maxRedirects: int = 3,
timeout: int = 1000, # in ms - 1 second
pinnedCert: string = "",
disallowHttp: bool = false,
userAgent: string = defUserAgent,
): HttpClient =
var context: SslContext

if uri.scheme in @["", "https"]:
context = getSSLContext(caFile = pinnedCert)
else:
if disallowHttp:
raise newException(ValueError, "http:// URLs not allowed (only https).")
elif pinnedCert != "":
raise newException(ValueError, "Pinned cert not allowed with http " &
"URL (only https).")

let client = newHttpClient(sslContext = context,
userAgent = userAgent,
timeout = timeout,
maxRedirects = maxRedirects)

if client == nil:
raise newException(ValueError, "Invalid HTTP configuration")

return client

proc safeRequest*(url: Uri | string,
httpMethod: HttpMethod | string = HttpGet,
body = "",
headers: HttpHeaders = nil,
multipart: MultipartData = nil,
retries: int = 0,
firstRetryDelayMs: int = 0,
timeout: int = 1000,
pinnedCert: string = "",
maxRedirects: int = 3,
disallowHttp: bool = false,
): Response =
var uri: Uri
when url is string:
uri = parseUri(url)
else:
uri = url
let client = createHttpClient(uri = uri,
maxRedirects = maxRedirects,
timeout = timeout,
pinnedCert = pinnedCert,
disallowHttp = disallowHttp)
return client.safeRequest(url = uri,
httpMethod = httpMethod,
body = body,
headers = headers,
multipart = multipart,
retries = retries,
firstRetryDelayMs = firstRetryDelayMs)
100 changes: 43 additions & 57 deletions nimutils/sinks.nim
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@

import streams, tables, options, os, strutils, std/[net, uri, httpclient],
s3client, pubsub, misc, random, encodings, std/tempfiles,
parseutils, openssl, file, std/asyncfutures, net
parseutils, file, std/asyncfutures, net

const defaultLogSearchPath = @["/var/log/", "~/.log/", "."]

Expand Down Expand Up @@ -304,10 +304,7 @@ proc s3SinkOut(msg: string, cfg: SinkConfig, t: Topic, ignored: StringTable) =
else:
cfg.iolog(t, "Post to: " & newPath & "; response = " & response.status)

proc SSL_CTX_load_verify_file(ctx: SslCtx, CAfile: cstring):
cint {.cdecl, dynlib: DLLSSLName, importc.}

proc httpHeadersForSink(cfg: SinkConfig): HttpHeaders =
proc httpHeaders(cfg: SinkConfig): HttpHeaders =
var
tups: seq[(string, string)] = @[]
contentType: string = cfg.params["content_type"]
Expand Down Expand Up @@ -336,16 +333,20 @@ proc httpHeadersForSink(cfg: SinkConfig): HttpHeaders =

return headers

template httpUriForSink(cfg: SinkConfig): Uri =
parseURI(cfg.params["uri"])

proc httpClientForSink(cfg: SinkConfig, uri: Uri, maxRedirects: int = 5): HttpClient =
proc httpParams(cfg: SinkConfig): tuple[
uri: Uri,
headers: HttpHeaders,
timeout: int,
disallowHttp: bool,
pinnedCert: string,
] =
let
uri = parseURI(cfg.params["uri"])
headers = cfg.httpHeaders()
disallowHttp = "disallow_http" in cfg.params
var
client: HttpClient
timeout: int
pinnedCert: string = ""
context: SslContext

timeout = 1000 # in ms - 1 second
pinnedCert = ""
if "pinned_cert_file" in cfg.params:
pinnedCert = cfg.params["pinned_cert_file"]
if "timeout" in cfg.params:
Expand All @@ -355,38 +356,20 @@ proc httpClientForSink(cfg: SinkConfig, uri: Uri, maxRedirects: int = 5): HttpCl
"represented as an integer, or 0 for no timeout.")
elif timeout <= 0:
timeout = -1
else:
timeout = 1000 # in ms - 1 second

if uri.scheme == "https":
context = newContext(verifyMode = CVerifyPeer)
if pinnedCert != "":
discard context.context.SSL_CTX_load_verify_file(cstring(pinnedCert))
client = newHttpClient(sslContext=context, timeout=timeout, maxRedirects = maxRedirects)
else:
if "disallow_http" in cfg.params:
raise newException(ValueError, "http:// URLs not allowed (only https).")
elif pinnedCert != "":
raise newException(ValueError, "Pinned cert not allowed with http " &
"URL (only https).")
client = newHttpClient(sslContext=nil, timeout=timeout, maxRedirects = maxRedirects)

if client == nil:
raise newException(ValueError, "Invalid HTTP configuration")

return client
return (uri, headers, timeout, disallowHttp, pinnedCert)

proc postSinkOut(msg: string, cfg: SinkConfig, t: Topic, ignored: StringTable) =
let
headers = httpHeadersForSink(cfg)
uri = httpUriForSink(cfg)
client = httpClientForSink(cfg, uri)
response = client.safeRequest(url = uri,
httpMethod = HttpPost,
body = msg,
headers = headers,
retries = 2,
firstRetryDelayMs = 100)
params = cfg.httpParams()
response = safeRequest(url = params.uri,
timeout = params.timeout,
headers = params.headers,
disallowHttp = params.disallowHttp,
pinnedCert = params.pinnedCert,
httpMethod = HttpPost,
body = msg,
retries = 2,
firstRetryDelayMs = 100)

if not response.code.is2xx():
raise newException(ValueError, response.status & ": " & response.body())
Expand All @@ -395,21 +378,22 @@ proc postSinkOut(msg: string, cfg: SinkConfig, t: Topic, ignored: StringTable) =

proc presignSinkOut(msg: string, cfg: SinkConfig, t: Topic, ignored: StringTable) =
let
headers = httpHeadersForSink(cfg)
signUri = httpUriForSink(cfg)
params = cfg.httpParams()
# for the sign request, we do not want to send full request payload as:
# * we expect a redirect response
# * server might not accept large requests (hence presigning sink is used)
# * no need to waste bandwidth
# and will only send it to the returned signed URL
# which is why we disallow redirects here via maxRedirects
# NOTE this assumes that the endpoint immediately returns presigned URL
signClient = httpClientForSink(cfg, signUri, maxRedirects = 0)
signResponse = signClient.safeRequest(url = signUri,
httpMethod = HttpPut,
headers = headers,
retries = 2,
firstRetryDelayMs = 100)
signResponse = safeRequest(url = params.uri,
timeout = params.timeout,
headers = params.headers,
disallowHttp = params.disallowHttp,
pinnedCert = params.pinnedCert,
httpMethod = HttpPut,
retries = 2,
firstRetryDelayMs = 100)

if signResponse.code notin [Http302, Http307]:
raise newException(ValueError, "Presign requires 302/307 redirect but received: " & signResponse.status)
Expand All @@ -423,12 +407,14 @@ proc presignSinkOut(msg: string, cfg: SinkConfig, t: Topic, ignored: StringTable
raise newException(ValueError, "Presign edirect Location header needs to be absolute URL")

let
client = httpClientForSink(cfg, uri)
response = client.safeRequest(url = uri,
httpMethod = HttpPut,
body = msg,
retries = 2,
firstRetryDelayMs = 100)
response = safeRequest(url = uri,
timeout = params.timeout,
disallowHttp = params.disallowHttp,
pinnedCert = params.pinnedCert,
httpMethod = HttpPut,
body = msg,
retries = 2,
firstRetryDelayMs = 100)

if not response.code.is2xx():
raise newException(ValueError, response.status & ": " & response.body())
Expand Down

0 comments on commit ebf0596

Please sign in to comment.