From ebf0596aa41ea4849ca0808a78cd62b4932d518a Mon Sep 17 00:00:00 2001 From: Miroslav Shubernetskiy Date: Thu, 8 Feb 2024 17:07:17 -0500 Subject: [PATCH] bundling mozilla root CA store for when there are no default certs not all machines have system certs such as in busybox containers and so by bundling certs from Mozilla we can fallback to them when system certs are missing --- nimutils/awsclient.nim | 5 +- nimutils/net.nim | 114 ++++++++++++++++++++++++++++++++++++++++- nimutils/sinks.nim | 100 ++++++++++++++++-------------------- 3 files changed, 158 insertions(+), 61 deletions(-) diff --git a/nimutils/awsclient.nim b/nimutils/awsclient.nim index fb195c1..37e383b 100644 --- a/nimutils/awsclient.nim +++ b/nimutils/awsclient.nim @@ -97,8 +97,9 @@ proc newAwsClient*(creds: AwsCredentials, region, service: string): AwsClient = let # TODO - use some kind of template and compile-time variable to put the correct kernel used to build the sdk in the UA? - httpclient = newHttpClient("nimaws-sdk/0.3.3; "&defUserAgent.replace(" ", - "-").toLower&"; darwin/16.7.0") + httpclient = createHttpClient( + userAgent = "nimaws-sdk/0.3.3; "&defUserAgent.replace(" ", "-").toLower&"; darwin/16.7.0", + ) scope = AwsScope(date: getAmzDateString(), region: region, service: service) return AwsClient(httpClient: httpclient, credentials: creds, scope: scope, diff --git a/nimutils/net.nim b/nimutils/net.nim index edaf18d..1ab5f23 100644 --- a/nimutils/net.nim +++ b/nimutils/net.nim @@ -1,4 +1,43 @@ -import std/[asyncfutures, net, httpclient, uri, math, os] +import std/[asyncfutures, net, httpclient, uri, math, os, streams, strutils] +import openssl +import ./managedtmp + +template getRootCAStoreContent(): string = + const + caWiki = "https://wiki.mozilla.org/CA/Included_Certificates" + caURL = "https://ccadb.my.salesforce-sites.com/mozilla/IncludedRootsPEMTxt?TrustBitsInclude=Websites" + curlCmd = "curl -fsS --retry 5 " & caURL + (contents, curlExitCode) = gorgeEx(curlCmd, cache="mozilla-root-store") + if curlExitCode != 0: + raise newException( + ValueError, + "Could not downlaod CA root store: " & contents + ) + const + opensslCmd = "openssl storeutl -noout -certs /dev/stdin" + (check, checkExitCode) = gorgeEx(opensslCmd, input=contents) + checkLines = check.splitLines() + if checkExitCode != 0: + raise newException( + ValueError, + "Could not validate CA root store certificates. " & + "Maybe server didnt return valid PEM file? " & + check + ) + echo("Embedding Mozilla Root CA store with certificates " & checkLines[^1].toLower()) + echo("For more information see " & caWiki) + contents + +var tmpCAStore = "" +proc getCAStorePath(): string = + const contents = getRootCAStoreContent() + if tmpCAStore != "": + return tmpCAStore + let (stream, tmp) = getNewTempFile("cabundle", ".pem") + stream.write(contents) + stream.close() + tmpCAStore = tmp + return tmp {.emit: """ #include @@ -9,7 +48,6 @@ import std/[asyncfutures, net, httpclient, uri, math, os] #include - // Cloudflare DNS const char * dummy_dst = "1.1.1.1"; const int dummy_port = 53; @@ -118,3 +156,75 @@ proc safeRequest*(client: HttpClient, withRetry(retries, firstRetryDelayMs): return client.request(url = url, httpMethod = httpMethod, body = body, headers = headers, multipart = multipart) + +# https://github.com/nim-lang/Nim/blob/a45f43da3407dbbf8ecd15ce8ecb361af677add7/lib/pure/httpclient.nim#L380-L386 +# similar to stdlib but defaults to bundled CAs +proc getSSLContext(caFile: string = ""): SslContext = + if caFile != "": + # note when caFile is provided there is no try..except + # otherwise we would silently fail to bundled CA root store + # if caFile is invalid/does not exist + return newContext(verifyMode = CVerifyPeer, caFile = caFile) + else: + try: + return newContext(verifyMode = CVerifyPeer) + except: + return newContext(verifyMode = CVerifyPeer, caFile = getCAStorePath()) + +proc createHttpClient*(uri: Uri = parseUri(""), + maxRedirects: int = 3, + timeout: int = 1000, # in ms - 1 second + pinnedCert: string = "", + disallowHttp: bool = false, + userAgent: string = defUserAgent, + ): HttpClient = + var context: SslContext + + if uri.scheme in @["", "https"]: + context = getSSLContext(caFile = pinnedCert) + else: + if disallowHttp: + raise newException(ValueError, "http:// URLs not allowed (only https).") + elif pinnedCert != "": + raise newException(ValueError, "Pinned cert not allowed with http " & + "URL (only https).") + + let client = newHttpClient(sslContext = context, + userAgent = userAgent, + timeout = timeout, + maxRedirects = maxRedirects) + + if client == nil: + raise newException(ValueError, "Invalid HTTP configuration") + + return client + +proc safeRequest*(url: Uri | string, + httpMethod: HttpMethod | string = HttpGet, + body = "", + headers: HttpHeaders = nil, + multipart: MultipartData = nil, + retries: int = 0, + firstRetryDelayMs: int = 0, + timeout: int = 1000, + pinnedCert: string = "", + maxRedirects: int = 3, + disallowHttp: bool = false, + ): Response = + var uri: Uri + when url is string: + uri = parseUri(url) + else: + uri = url + let client = createHttpClient(uri = uri, + maxRedirects = maxRedirects, + timeout = timeout, + pinnedCert = pinnedCert, + disallowHttp = disallowHttp) + return client.safeRequest(url = uri, + httpMethod = httpMethod, + body = body, + headers = headers, + multipart = multipart, + retries = retries, + firstRetryDelayMs = firstRetryDelayMs) diff --git a/nimutils/sinks.nim b/nimutils/sinks.nim index 1f8ab13..c1b2652 100644 --- a/nimutils/sinks.nim +++ b/nimutils/sinks.nim @@ -3,7 +3,7 @@ import streams, tables, options, os, strutils, std/[net, uri, httpclient], s3client, pubsub, misc, random, encodings, std/tempfiles, - parseutils, openssl, file, std/asyncfutures, net + parseutils, file, std/asyncfutures, net const defaultLogSearchPath = @["/var/log/", "~/.log/", "."] @@ -304,10 +304,7 @@ proc s3SinkOut(msg: string, cfg: SinkConfig, t: Topic, ignored: StringTable) = else: cfg.iolog(t, "Post to: " & newPath & "; response = " & response.status) -proc SSL_CTX_load_verify_file(ctx: SslCtx, CAfile: cstring): - cint {.cdecl, dynlib: DLLSSLName, importc.} - -proc httpHeadersForSink(cfg: SinkConfig): HttpHeaders = +proc httpHeaders(cfg: SinkConfig): HttpHeaders = var tups: seq[(string, string)] = @[] contentType: string = cfg.params["content_type"] @@ -336,16 +333,20 @@ proc httpHeadersForSink(cfg: SinkConfig): HttpHeaders = return headers -template httpUriForSink(cfg: SinkConfig): Uri = - parseURI(cfg.params["uri"]) - -proc httpClientForSink(cfg: SinkConfig, uri: Uri, maxRedirects: int = 5): HttpClient = +proc httpParams(cfg: SinkConfig): tuple[ + uri: Uri, + headers: HttpHeaders, + timeout: int, + disallowHttp: bool, + pinnedCert: string, +] = + let + uri = parseURI(cfg.params["uri"]) + headers = cfg.httpHeaders() + disallowHttp = "disallow_http" in cfg.params var - client: HttpClient - timeout: int - pinnedCert: string = "" - context: SslContext - + timeout = 1000 # in ms - 1 second + pinnedCert = "" if "pinned_cert_file" in cfg.params: pinnedCert = cfg.params["pinned_cert_file"] if "timeout" in cfg.params: @@ -355,38 +356,20 @@ proc httpClientForSink(cfg: SinkConfig, uri: Uri, maxRedirects: int = 5): HttpCl "represented as an integer, or 0 for no timeout.") elif timeout <= 0: timeout = -1 - else: - timeout = 1000 # in ms - 1 second - - if uri.scheme == "https": - context = newContext(verifyMode = CVerifyPeer) - if pinnedCert != "": - discard context.context.SSL_CTX_load_verify_file(cstring(pinnedCert)) - client = newHttpClient(sslContext=context, timeout=timeout, maxRedirects = maxRedirects) - else: - if "disallow_http" in cfg.params: - raise newException(ValueError, "http:// URLs not allowed (only https).") - elif pinnedCert != "": - raise newException(ValueError, "Pinned cert not allowed with http " & - "URL (only https).") - client = newHttpClient(sslContext=nil, timeout=timeout, maxRedirects = maxRedirects) - - if client == nil: - raise newException(ValueError, "Invalid HTTP configuration") - - return client + return (uri, headers, timeout, disallowHttp, pinnedCert) proc postSinkOut(msg: string, cfg: SinkConfig, t: Topic, ignored: StringTable) = let - headers = httpHeadersForSink(cfg) - uri = httpUriForSink(cfg) - client = httpClientForSink(cfg, uri) - response = client.safeRequest(url = uri, - httpMethod = HttpPost, - body = msg, - headers = headers, - retries = 2, - firstRetryDelayMs = 100) + params = cfg.httpParams() + response = safeRequest(url = params.uri, + timeout = params.timeout, + headers = params.headers, + disallowHttp = params.disallowHttp, + pinnedCert = params.pinnedCert, + httpMethod = HttpPost, + body = msg, + retries = 2, + firstRetryDelayMs = 100) if not response.code.is2xx(): raise newException(ValueError, response.status & ": " & response.body()) @@ -395,8 +378,7 @@ proc postSinkOut(msg: string, cfg: SinkConfig, t: Topic, ignored: StringTable) = proc presignSinkOut(msg: string, cfg: SinkConfig, t: Topic, ignored: StringTable) = let - headers = httpHeadersForSink(cfg) - signUri = httpUriForSink(cfg) + params = cfg.httpParams() # for the sign request, we do not want to send full request payload as: # * we expect a redirect response # * server might not accept large requests (hence presigning sink is used) @@ -404,12 +386,14 @@ proc presignSinkOut(msg: string, cfg: SinkConfig, t: Topic, ignored: StringTable # and will only send it to the returned signed URL # which is why we disallow redirects here via maxRedirects # NOTE this assumes that the endpoint immediately returns presigned URL - signClient = httpClientForSink(cfg, signUri, maxRedirects = 0) - signResponse = signClient.safeRequest(url = signUri, - httpMethod = HttpPut, - headers = headers, - retries = 2, - firstRetryDelayMs = 100) + signResponse = safeRequest(url = params.uri, + timeout = params.timeout, + headers = params.headers, + disallowHttp = params.disallowHttp, + pinnedCert = params.pinnedCert, + httpMethod = HttpPut, + retries = 2, + firstRetryDelayMs = 100) if signResponse.code notin [Http302, Http307]: raise newException(ValueError, "Presign requires 302/307 redirect but received: " & signResponse.status) @@ -423,12 +407,14 @@ proc presignSinkOut(msg: string, cfg: SinkConfig, t: Topic, ignored: StringTable raise newException(ValueError, "Presign edirect Location header needs to be absolute URL") let - client = httpClientForSink(cfg, uri) - response = client.safeRequest(url = uri, - httpMethod = HttpPut, - body = msg, - retries = 2, - firstRetryDelayMs = 100) + response = safeRequest(url = uri, + timeout = params.timeout, + disallowHttp = params.disallowHttp, + pinnedCert = params.pinnedCert, + httpMethod = HttpPut, + body = msg, + retries = 2, + firstRetryDelayMs = 100) if not response.code.is2xx(): raise newException(ValueError, response.status & ": " & response.body())