diff --git a/.gitignore b/.gitignore index 6e92f57..b243841 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ tags +nimutils.out diff --git a/bin/buildlibs.sh b/bin/buildlibs.sh new file mode 100755 index 0000000..15269a9 --- /dev/null +++ b/bin/buildlibs.sh @@ -0,0 +1,298 @@ +#!/bin/bash + +function color { + case $1 in + black) CODE=0 ;; + red) CODE=1 ;; RED) CODE=9 ;; + green) CODE=2 ;; GREEN) CODE=10 ;; + yellow) CODE=3 ;; YELLOW) CODE=11 ;; + blue) CODE=4 ;; BLUE) CODE=12 ;; + magenta) CODE=5 ;; MAGENTA) CODE=13 ;; + cyan) CODE=6 ;; CYAN) CODE=14 ;; + white) CODE=7 ;; WHITE) CODE=15 ;; + grey) CODE=8 ;; *) CODE=$1 ;; + esac + shift + + export TERM=${TERM:-vt100} + echo -n $(tput -T ${TERM} setaf ${CODE})$@$(tput -T ${TERM} op) +} + +function colorln { + echo $(color $@) +} + +if [[ ${#} -eq 0 ]] ; then + colorln RED Script requires an argument pointing to the deps directory + exit 1 +fi + +ARCH=$(uname -m) +OS=$(uname -o 2>/dev/null) +if [[ $? -ne 0 ]] ; then + # Older macOS/OSX versions of uname don't support -o + OS=$(uname -s) +fi + +if [[ ${OS} = "Darwin" ]] ; then + # Not awesome, but this is what nim calls it. + OS=macosx + + # We might be running virtualized, so do some more definitive + # tesitng. Note that there's no cross-compiling flag; if you + # want to cross compile, you currently need to manually build + # these libs. + SYSCTL=$(sysctl -n sysctl.proc_translated 2>/dev/null) + if [[ ${SYSCTL} = '0' ]] || [[ ${SYSCTL} == '1' ]] ; then + NIMARCH=arm64 + else + NIMARCH=amd64 + fi +else + # We don't support anything else at the moment. + OS=linux + if [[ ${ARCH} = "x86_64" ]] ; then + NIMARCH=amd64 + else + NIMARCH=arm64 + fi +fi + +DEPS_DIR=${DEPS_DIR:-${HOME}/.local/c0} + +PKG_LIBS=${1}/lib/${OS}-${NIMARCH} +MY_LIBS=${DEPS_DIR}/libs +SRC_DIR=${DEPS_DIR}/src +MUSL_DIR=${DEPS_DIR}/musl +MUSL_GCC=${MUSL_DIR}/bin/musl-gcc + +mkdir -p ${MY_LIBS} + +# The paste doesn't work from stdin on MacOS, so leave this as is, please. +export OPENSSL_CONFIG_OPTS=$(echo " +enable-ec_nistp_64_gcc_128 +no-afalgeng +no-apps +no-bf +no-camellia +no-cast +no-comp +no-deprecated +no-des +no-docs +no-dtls +no-dtls1 +no-egd +no-engine +no-err +no-idea +no-md2 +no-md4 +no-mdc2 +no-psk +no-quic +no-rc2 +no-rc4 +no-rc5 +no-seed +no-shared +no-srp +no-ssl +no-tests +no-tls1 +no-tls1_1 +no-uplink +no-weak-ssl-ciphers +no-zlib +" | tr '\n' ' ') + +function copy_from_package { + for item in ${@} + do + if [[ ! -f ${MY_LIBS}/${item} ]] ; then + if [[ ! -f ${PKG_LIBS}/${item} ]] ; then + return 1 + else + cp ${PKG_LIBS}/${item} ${MY_LIBS} + echo $(color GREEN Installed ${item} to:) ${MY_LIBS} + fi + fi + done + return 0 +} + +function get_src { + mkdir -p ${SRC_DIR} + cd ${SRC_DIR} + + if [[ ! -d ${SRC_DIR}/${1} ]] ; then + echo $(color CYAN Downloading ${1} from:) ${2} + git clone ${2} + fi + if [[ ! -d ${1} ]] ; then + echo $(color RED Could not create directory: ) ${SRC_DIR}/${1} + exit 1 + fi + cd ${1} +} + +function ensure_musl { + if [[ ${OS} = "macosx" ]] ; then + return + fi + if [[ ! -f ${MUSL_GCC} ]] ; then + # if musl-gcc is already installed, use it + existing_musl=$(which musl-gcc 2> /dev/null) + if [[ -n "${existing_musl}" ]]; then + mkdir -p $(dirname ${MUSL_GCC}) + ln -s ${existing_musl} ${MUSL_GCC} + echo $(color GREEN Linking existing musl-gcc: ) ${existing_musl} $(color GREEN "->" ) ${MUSL_GCC} + fi + fi + if [[ ! -f ${MUSL_GCC} ]] ; then + get_src musl git://git.musl-libc.org/musl + colorln CYAN Building musl + unset CC + ./configure --disable-shared --prefix=${MUSL_DIR} + make clean + make + make install + mv lib/*.a ${MY_LIBS} + + if [[ -f ${MUSL_GCC} ]] ; then + echo $(color GREEN Installed musl wrapper to:) ${MUSL_GCC} + else + colorln RED Installation of musl failed! + exit 1 + fi + fi + export CC=${MUSL_GCC} + export CXX=${MUSL_GCC} +} + +function install_kernel_headers { + if [[ ${OS} = "macosx" ]] ; then + return + fi + colorln CYAN Installing kernel headers needed for musl install + get_src kernel-headers https://github.com/sabotage-linux/kernel-headers.git + make ARCH=${ARCH} prefix= DESTDIR=${MUSL_DIR} install +} + +function ensure_openssl { + + if ! copy_from_package libssl.a libcrypto.a ; then + ensure_musl + install_kernel_headers + + get_src openssl https://github.com/openssl/openssl.git + colorln CYAN Building openssl + if [[ ${OS} == "macosx" ]]; then + ./config ${OPENSSL_CONFIG_OPTS} + else + ./config ${OPENSSL_CONFIG_OPTS} -static + fi + make clean + make build_libs + mv *.a ${MY_LIBS} + if [[ -f ${MY_LIBS}/libssl.a ]] && [[ -f ${MY_LIBS}/libcrypto.a ]] ; then + echo $(color GREEN Installed openssl libs to:) ${MY_LIBS} + else + colorln RED Installation of openssl failed! + exit 1 + fi + fi +} + +function ensure_pcre { + if ! copy_from_package libpcre.a ; then + + get_src pcre https://github.com/luvit/pcre.git + colorln CYAN "Building libpcre" + # For some reason, build fails on arm if we try to compile w/ musl? + unset CC + ./configure --disable-cpp --disable-shared + make clean + make + + mv .libs/libpcre.a ${MY_LIBS} + if [[ -f ${MY_LIBS}/libpcre.a ]] ; then + echo $(color GREEN Installed libpcre to:) ${MY_LIBS}/libpcre.a + else + colorln RED "Installation of libprce failed. This may be due to missing build dependencies. Please make sure autoconf, m4 and perl are installed." + exit 1 + fi + fi +} + +function ensure_gumbo { + if ! copy_from_package libgumbo.a ; then + ensure_musl + get_src sigil-gumbo https://github.com/Sigil-Ebook/sigil-gumbo/ + colorln CYAN "Watching our waistline, selecting only required gumbo ingredients..." + cat > CMakeLists.txt < /dev/null + echo `pwd` + popd > /dev/null +} + +SRC_DIR=$(to_abs_dir ${NIMUTILS_DIR:-.}/nimutils/c) +DST_DIR=${HEADER_DIR:-~/.local/c0/include} + +mkdir -p $DST_DIR + +DST_DIR=$(to_abs_dir $DST_DIR) + +function copy_news { + # $1 -- source directory + # $2 -- destination directory + # $3 -- file name. + + SRC_FILE=$1/$3 + DST_FILE=$2/$3 + + if [[ $SRC_FILE -nt $DST_FILE ]]; then + if [[ ! -e $SRC_FILE ]]; then + echo $(color RED error:) specified 'cp ${SRC_FILE} ${DST_FILE}' but the source file does not exist. + else + if [[ ! -e $DST_FILE ]]; then + echo $(color YELLOW "Copying new file: ") $3 + echo $(color YELLOW to: ) $DST_FILE + else + echo $(color GREEN "Updating file:" ) $3 + echo $(color GREEN full location: $DST_FILE) + fi + cp $SRC_FILE $DST_FILE + fi + fi +} + +function push_ext_files { + # $1 is the src dir + # $2 is the dst dir + # $3 is the extension + pushd $1 >/dev/null + + for item in `ls *.$3`; do + copy_news $1 $2 $item + done + + popd >/dev/null +} + +push_ext_files ${SRC_DIR} ${DST_DIR} h diff --git a/config.nims b/config.nims index 512f4bd..4e3e545 100644 --- a/config.nims +++ b/config.nims @@ -1,5 +1,31 @@ -switch("d", "ssl") -switch("d", "nimPreviewHashRef") -switch("gc", "refc") -switch("path", ".") -#switch("d", "release") +import strutils, os, nimutils/nimscript + +when (NimMajor, NimMinor) < (2, 0): + echo "NimUtils requires Nim 2.0 or later." + quit() + +when not defined(debug): + switch("d", "release") + switch("opt", "speed") + +var + subdir = "" + +for item in listDirs(thisDir()): + if item.endswith("/files"): + subdir = "/files" + break + +proc getEnvDir(s: string, default = ""): string = + result = getEnv(s, default) + +exec thisDir() & "/bin/buildlibs.sh " & thisDir() & "/files/deps" + +var + default = getEnvDir("HOME").joinPath(".local/c0") + localDir = getEnvDir("LOCAL_INSTALL_DIR", default) + libDir = localdir.joinPath("libs") + libs = ["pcre", "ssl", "crypto", "gumbo", "hatrack"] + +applyCommonLinkOptions() +staticLinkLibraries(libs, libDir, muslBase = localDir) diff --git a/files/deps/lib/linux-amd64/libc.a b/files/deps/lib/linux-amd64/libc.a new file mode 100644 index 0000000..3617210 Binary files /dev/null and b/files/deps/lib/linux-amd64/libc.a differ diff --git a/files/deps/lib/linux-amd64/libcrypt.a b/files/deps/lib/linux-amd64/libcrypt.a new file mode 100644 index 0000000..8b277f0 --- /dev/null +++ b/files/deps/lib/linux-amd64/libcrypt.a @@ -0,0 +1 @@ +! diff --git a/files/deps/lib/linux-amd64/libcrypto.a b/files/deps/lib/linux-amd64/libcrypto.a new file mode 100644 index 0000000..d5569dd Binary files /dev/null and b/files/deps/lib/linux-amd64/libcrypto.a differ diff --git a/files/deps/lib/linux-amd64/libdl.a b/files/deps/lib/linux-amd64/libdl.a new file mode 100644 index 0000000..8b277f0 --- /dev/null +++ b/files/deps/lib/linux-amd64/libdl.a @@ -0,0 +1 @@ +! diff --git a/files/deps/lib/linux-amd64/libgumbo.a b/files/deps/lib/linux-amd64/libgumbo.a new file mode 100644 index 0000000..7dd0608 Binary files /dev/null and b/files/deps/lib/linux-amd64/libgumbo.a differ diff --git a/files/deps/lib/linux-amd64/libm.a b/files/deps/lib/linux-amd64/libm.a new file mode 100644 index 0000000..8b277f0 --- /dev/null +++ b/files/deps/lib/linux-amd64/libm.a @@ -0,0 +1 @@ +! diff --git a/files/deps/lib/linux-amd64/libpcre.a b/files/deps/lib/linux-amd64/libpcre.a new file mode 100644 index 0000000..94abf2d Binary files /dev/null and b/files/deps/lib/linux-amd64/libpcre.a differ diff --git a/files/deps/lib/linux-amd64/libpthread.a b/files/deps/lib/linux-amd64/libpthread.a new file mode 100644 index 0000000..8b277f0 --- /dev/null +++ b/files/deps/lib/linux-amd64/libpthread.a @@ -0,0 +1 @@ +! diff --git a/files/deps/lib/linux-amd64/libresolv.a b/files/deps/lib/linux-amd64/libresolv.a new file mode 100644 index 0000000..8b277f0 --- /dev/null +++ b/files/deps/lib/linux-amd64/libresolv.a @@ -0,0 +1 @@ +! diff --git a/files/deps/lib/linux-amd64/librt.a b/files/deps/lib/linux-amd64/librt.a new file mode 100644 index 0000000..8b277f0 --- /dev/null +++ b/files/deps/lib/linux-amd64/librt.a @@ -0,0 +1 @@ +! diff --git a/files/deps/lib/linux-amd64/libssl.a b/files/deps/lib/linux-amd64/libssl.a new file mode 100644 index 0000000..e0819d3 Binary files /dev/null and b/files/deps/lib/linux-amd64/libssl.a differ diff --git a/files/deps/lib/linux-amd64/libutil.a b/files/deps/lib/linux-amd64/libutil.a new file mode 100644 index 0000000..8b277f0 --- /dev/null +++ b/files/deps/lib/linux-amd64/libutil.a @@ -0,0 +1 @@ +! diff --git a/files/deps/lib/linux-amd64/libxnet.a b/files/deps/lib/linux-amd64/libxnet.a new file mode 100644 index 0000000..8b277f0 --- /dev/null +++ b/files/deps/lib/linux-amd64/libxnet.a @@ -0,0 +1 @@ +! diff --git a/files/deps/lib/linux-arm64/libc.a b/files/deps/lib/linux-arm64/libc.a new file mode 100644 index 0000000..c69da4a Binary files /dev/null and b/files/deps/lib/linux-arm64/libc.a differ diff --git a/files/deps/lib/linux-arm64/libcrypt.a b/files/deps/lib/linux-arm64/libcrypt.a new file mode 100644 index 0000000..8b277f0 --- /dev/null +++ b/files/deps/lib/linux-arm64/libcrypt.a @@ -0,0 +1 @@ +! diff --git a/files/deps/lib/linux-arm64/libcrypto.a b/files/deps/lib/linux-arm64/libcrypto.a new file mode 100644 index 0000000..aafb92f Binary files /dev/null and b/files/deps/lib/linux-arm64/libcrypto.a differ diff --git a/files/deps/lib/linux-arm64/libdl.a b/files/deps/lib/linux-arm64/libdl.a new file mode 100644 index 0000000..8b277f0 --- /dev/null +++ b/files/deps/lib/linux-arm64/libdl.a @@ -0,0 +1 @@ +! diff --git a/files/deps/lib/linux-arm64/libgumbo.a b/files/deps/lib/linux-arm64/libgumbo.a new file mode 100644 index 0000000..788196e Binary files /dev/null and b/files/deps/lib/linux-arm64/libgumbo.a differ diff --git a/files/deps/lib/linux-arm64/libm.a b/files/deps/lib/linux-arm64/libm.a new file mode 100644 index 0000000..8b277f0 --- /dev/null +++ b/files/deps/lib/linux-arm64/libm.a @@ -0,0 +1 @@ +! diff --git a/files/deps/lib/linux-arm64/libpcre.a b/files/deps/lib/linux-arm64/libpcre.a new file mode 100644 index 0000000..44cb639 Binary files /dev/null and b/files/deps/lib/linux-arm64/libpcre.a differ diff --git a/files/deps/lib/linux-arm64/libpthread.a b/files/deps/lib/linux-arm64/libpthread.a new file mode 100644 index 0000000..8b277f0 --- /dev/null +++ b/files/deps/lib/linux-arm64/libpthread.a @@ -0,0 +1 @@ +! diff --git a/files/deps/lib/linux-arm64/libresolv.a b/files/deps/lib/linux-arm64/libresolv.a new file mode 100644 index 0000000..8b277f0 --- /dev/null +++ b/files/deps/lib/linux-arm64/libresolv.a @@ -0,0 +1 @@ +! diff --git a/files/deps/lib/linux-arm64/librt.a b/files/deps/lib/linux-arm64/librt.a new file mode 100644 index 0000000..8b277f0 --- /dev/null +++ b/files/deps/lib/linux-arm64/librt.a @@ -0,0 +1 @@ +! diff --git a/files/deps/lib/linux-arm64/libssl.a b/files/deps/lib/linux-arm64/libssl.a new file mode 100644 index 0000000..71cdced Binary files /dev/null and b/files/deps/lib/linux-arm64/libssl.a differ diff --git a/files/deps/lib/linux-arm64/libutil.a b/files/deps/lib/linux-arm64/libutil.a new file mode 100644 index 0000000..8b277f0 --- /dev/null +++ b/files/deps/lib/linux-arm64/libutil.a @@ -0,0 +1 @@ +! diff --git a/files/deps/lib/linux-arm64/libxnet.a b/files/deps/lib/linux-arm64/libxnet.a new file mode 100644 index 0000000..8b277f0 --- /dev/null +++ b/files/deps/lib/linux-arm64/libxnet.a @@ -0,0 +1 @@ +! diff --git a/files/deps/lib/macosx-amd64/libcrypto.a b/files/deps/lib/macosx-amd64/libcrypto.a new file mode 100644 index 0000000..dec3910 Binary files /dev/null and b/files/deps/lib/macosx-amd64/libcrypto.a differ diff --git a/files/deps/lib/macosx-amd64/libpcre.a b/files/deps/lib/macosx-amd64/libpcre.a new file mode 100644 index 0000000..ca0f488 Binary files /dev/null and b/files/deps/lib/macosx-amd64/libpcre.a differ diff --git a/files/deps/lib/macosx-amd64/libssl.a b/files/deps/lib/macosx-amd64/libssl.a new file mode 100644 index 0000000..fb404ef Binary files /dev/null and b/files/deps/lib/macosx-amd64/libssl.a differ diff --git a/files/deps/lib/macosx-arm64/libcrypto.a b/files/deps/lib/macosx-arm64/libcrypto.a new file mode 100644 index 0000000..9218027 Binary files /dev/null and b/files/deps/lib/macosx-arm64/libcrypto.a differ diff --git a/files/deps/lib/macosx-arm64/libgumbo.a b/files/deps/lib/macosx-arm64/libgumbo.a new file mode 100644 index 0000000..a1a5521 Binary files /dev/null and b/files/deps/lib/macosx-arm64/libgumbo.a differ diff --git a/files/deps/lib/macosx-arm64/libhatrack.a b/files/deps/lib/macosx-arm64/libhatrack.a new file mode 100644 index 0000000..9ad5822 Binary files /dev/null and b/files/deps/lib/macosx-arm64/libhatrack.a differ diff --git a/files/deps/lib/macosx-arm64/libpcre.a b/files/deps/lib/macosx-arm64/libpcre.a new file mode 100644 index 0000000..9d4195a Binary files /dev/null and b/files/deps/lib/macosx-arm64/libpcre.a differ diff --git a/files/deps/lib/macosx-arm64/libssl.a b/files/deps/lib/macosx-arm64/libssl.a new file mode 100644 index 0000000..392a81c Binary files /dev/null and b/files/deps/lib/macosx-arm64/libssl.a differ diff --git a/nimutils.nim b/nimutils.nim index f0da417..10bbe69 100644 --- a/nimutils.nim +++ b/nimutils.nim @@ -11,16 +11,20 @@ ## few fixes have all been for compatability and are made under the ## same license. I also migrated the crypto to openssl. -import nimutils/[box, random, unicodeid, pubsub, sinks, misc, texttable], - nimutils/[file, process, filetable, encodings, advisory_lock, progress], +import nimutils/[box, random, unicodeid, pubsub, sinks, misc, texttable, dict], + nimutils/[file, filetable, encodings, advisory_lock, progress], nimutils/[sha, aes, prp, hexdump, markdown, htmlparse, net], nimutils/[colortable, rope_base, rope_styles, rope_construct], nimutils/[rope_prerender, rope_ansirender, switchboard, subproc] export box, random, unicodeid, pubsub, sinks, misc, random, texttable, - file, process, filetable, encodings, advisory_lock, progress, sha, + file, filetable, encodings, advisory_lock, progress, sha, aes, prp, hexdump, markdown, htmlparse, net, colortable, rope_base, rope_styles, rope_construct, rope_prerender, rope_ansirender, - switchboard, subproc + switchboard, subproc, dict + +when defined(macosx): + import nimutils/macproc + export macproc ## Things we don't want to force people to consume need to be imported ## manually. Currently, that's: @@ -30,3 +34,417 @@ export box, random, unicodeid, pubsub, sinks, misc, random, texttable, ## `managedtmp` because it adds a destructor you might not want. ## `randwords` because it does have a huge data structure embedded, which ## isn't worth it if you're not using it. + +when isMainModule: + import tables, streams, algorithm, strutils, unicode + + when defined(macosx): + proc psSorter(x, y: ProcessInfo): int = + return cmp(x.pid, y.pid) + + proc macProcTest() = + print("

Mac proc test

") + var + psInfo = listProcesses() + cap = h3("found " & $(psInfo.len()) & " processes.") + head = @[tr(@[th("Pid"), th("Command"), th("Arguments"), th("User")])] + widths = colPcts([10, 35, 40, 15]) + + psInfo.sort(psSorter) + var body: seq[Rope] + for pr in psInfo: + body.add(tr(@[td(fgColor($(pr.getPid()), "blue")), + td(pr.getExePath()), + td(pr.getArgv().join(" ")), + td(fgColor(pr.getUserName(), "fandango"))])) + + print(table(tbody(body), thead = thead(head), caption = cap, + columnInfo = widths)) + + proc basic_subproc_tests() = + print(h2("Run: /bin/cat /etc/passwd /etc/file_that_doesnt_exist; show output.")) + let res = runCmdGetEverything("/bin/cat", @["/etc/passwd", + "/etc/file_that_doesnt_exist"], + passthrough = true) + print(fgColor("PID was: ", "atomiclime") + em($(res.getPid()))) + print(fgColor("Exit code was: ", "atomiclime") + em($(res.getExit()))) + print(fgColor("Stdout was: ", "atomiclime") + em(pre(res.getStdout()))) + print(fgColor("Stderr was: ", "atomiclime") + text(res.getStderr())) + print(strdump(res.getStderr())) + + proc boxTest() = + print("

Box tests

") + var + i1 = "a" + l1 = @["a", "b", "c"] + l2 = @["d", "e", "f"] + l3 = @["g", "h", "i"] + l123 = @[l1, l2, l3] + b1, b123: Box + o123: seq[seq[string]] = @[] + oMy: seq[Box] = @[] + a1 = pack(i1) + + echo typeof(a1) + echo unpack[string](a1) + b1 = pack(l1) + echo b1 + echo unpack[seq[string]](b1) + b123 = pack(l123) + echo b123 + echo typeof(b123) + echo typeof(o123) + o123 = unpack[seq[seq[string]]](b123) + echo o123 + oMy = unpack[seq[Box]](b123) + echo oMy + + var myDict = newTable[string, seq[string]]() + + myDict["foo"] = @["a", "b"] + myDict["bar"] = @["b"] + myDict["boz"] = @["c"] + myDict["you"] = @["d"] + let + f = newFileStream("nimutils.nim", fmRead) + contents = f.readAll()[0 .. 20] + + myDict["file"] = @[contents] + + let + dictBox = pack(myDict) + listbox = pack(l1) + + var outlist: l1.type + unpack(listbox, outlist) + + echo "Here's the listbox: ", listbox + echo "Here it is unpacked: ", outlist + + var newDict: TableRef[string, seq[string]] + + unpack(dictBox, newDict) + + echo "Here's the dictbox(nothing should be quoted): ", dictBox + echo "Here it is unpacked (should have quotes): ", newDict + echo "Here it is, boxed, as Json: ", boxToJson(dictBox) + + # This shouldn't work w/o a custom handler. + # import sugar + # var v: ()->int + # unpack[()->int](b123, v) + + proc ulidTests() = + print(h2("Ulid Encode / decode tests")) + let x = getUlid() + echo unixTimeInMs() + echo x, " ", x.ulidToTimeStamp() + let y = getUlid() + echo y, " ", y.ulidToTimeStamp() + echo unixTimeInMs() + echo base32Encode("This is some string.") + echo "KRUGS4ZANFZSA43PNVSSA43UOJUW4ZZO (is the answer)" + echo base32Encode("This is some string") + echo "KRUGS4ZANFZSA43PNVSSA43UOJUW4ZY (is the answer)" + echo base32Encode("This is some strin") + echo "KRUGS4ZANFZSA43PNVSSA43UOJUW4 (is the answer)" + echo base32Encode("This is some stri") + echo "KRUGS4ZANFZSA43PNVSSA43UOJUQ (is the answer)" + echo base32Encode("This is some str") + echo "KRUGS4ZANFZSA43PNVSSA43UOI (is the answer)" + + + echo "-----" + echo base32vEncode("This is some string.") + echo base32vDecode(base32vEncode("1his is some string.")) + echo base32vEncode("This is some string") + echo base32vDecode(base32vEncode("2his is some string")) + echo base32vEncode("This is some strin") + echo base32vDecode(base32vEncode("3his is some strin")) + echo base32vEncode("This is some stri") + echo base32vDecode(base32vEncode("4his is some stri")) + echo base32vEncode("This is some str") + echo base32vDecode(base32vEncode("5his is some str")) + + echo "-----" + echo base32Encode("This is some string.") + echo base32Decode(base32Encode("1his is some string.")) + echo base32Encode("This is some string") + echo base32Decode(base32Encode("2his is some string")) + echo base32Encode("This is some strin") + echo base32Decode(base32Encode("3his is some strin")) + echo base32Encode("This is some stri") + echo base32Decode(base32Encode("4his is some stri")) + echo base32Encode("This is some str") + echo base32Decode(base32Encode("5his is some str")) + + proc hexTests() = + var buf: array[128, byte] = [ + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, + 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, + 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, + 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, + 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, + 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, + 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, + 123, 124, 125, 126, 127 ] + + print(h2("Basic hexdump")) + print(pre(hexDump(listAddr(buf), 128, width = 80))) + + proc prpTests() = + print(h2("Luby-Rackoff PRP")) + var + nonce: string = "" + key = "0123456789abcdef" + pt = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" & + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ct = prp(key, pt, nonce) + pt2 = brb(key, ct, nonce) + + print(h3("Plaintext:")) + print(pre(strdump(pt))) + print(pre("Encrypt:")) + print(pre(strdump(ct))) + print(h3("Decrypt:")) + print(pre(strdump(pt2))) + print(h2("Random number generation")) + let rows = @[ + tr(@[th("uint64"), td($(secureRand[uint64]()))]), + tr(@[th("uint32"), td($(secureRand[uint32]()))]), + tr(@[th("float"), td($(secureRand[float]()))]), + tr(@[th("array[6, byte]"), td($(secureRand[array[6, byte]]()))]), + tr(@[th("string of 12 bytes (hex encoded)"), + td($(randString(12).hex()))])] + + print(table(tbody(rows))) + + proc shaTests() = + print(h3("SHA-256")) + var + text = newFileStream("nimutils.nim").readAll() + ctx: Sha256ctx + + initSha256(ctx) + ctx.update(text) + print(em(ctx.final().hex())) + print(em(sha256(text).hex())) + print(em(hmacSha256("foo", "bar").hex())) + print(em(hmacSha256Hex("foo", "bar"))) + + proc aesGcmTests() = + print(h3("AES")) + var + encCtx: GcmCtx + decCtx: GcmCtx + pt = "This is a test between disco and death" + key = "0123456789abcdef" + gcmInitEncrypt(encCtx, key) + gcmInitDecrypt(decCtx, key) + + print(h3("Initial pt:")) + print(em(pt)) + + for i in 1 .. 3: + var + ct = encCtx.gcmEncrypt(pt) + nonce = encCtx.gcmGetNonce() + pt = decCtx.gcmDecrypt(ct, nonce).get("") + + print(h3("Nonce:")) + print(em(nonce.hex())) + print(h3("CT: ")) + print(pre(strDump(ct))) + print(h3("Decrypted:")) + print(em(pt)) + + proc keyStreamTest() = + print(h2("Keystream test")) + + let + key = "0123456789abcdef" + stream1 = aesPrfOneShot(key, 200) + stream2 = aesPrfOneShot(key, 200) + + print(code(stream1.hex())) + assert len(stream1) == 200 + assert stream1 == stream2 + + var text = "This is a test, yo, dawg" + + aesCtrInPlaceOneshot(key, text) + + print(h3("PT:")) + print(em(text.hex())) + + aesCtrInPlaceOneshot(key, text) + + print(h3("Decrypted:")) + print(em(text)) + + proc dictTests() = + print(h2("Dictionary tests")) + + var + x: DictRef[int, string] = {42: "bar", 1000 : "zork", 17 : "foo", + 500: "boz"}.toDict() + y: Dict[int, string] + + y[500] = "boz" + y[1000] = "zork" + y[17] = "foo" + y[42] = "bar" + + echo x[42] + echo x[17] + x[17] = "blah" + y[17] = "blah" + echo x[17] + for i in 1..1000: + x[17] = $i + y[17] = x[17] + if i mod 2 == 1: + x.del(17) + y.del(17) + + echo x.keys() + echo x.values() + echo x.items() + + echo y.keys() + echo y.values() + echo y.items() + var d2: DictRef[string, int] = newDict[string, int]() + var seqstr = ["ay", "bee", "cee", "dee", "e", "eff", "gee", "h", "i", "j"] + + for i, item in seqstr: + d2[item] = i + + echo d2.keys() + echo d2.values() + echo d2.items() + echo d2.keys(sort = true) + echo d2.values(sort = true) + echo d2.items(sort = true) + + var d3 = newDict[string, array[10, string]]() + for item in seqstr: + d3[item] = seqstr + + echo d3[seqstr[0]] + + echo x + echo y + echo d2 + echo d3 + + proc instantTableTests() = + print(h2("Instant table tests")) + var mess1 = @["a.out.dSYM", "encodings.nim", "managedtmp.nim", + "random.nim", "sinks.nim", "advisory_lock.nim", "file.nim", + "markdown.nim", "randwords.nim", "subproc.c", + "aes.nim", "filetable.nim", "misc", "rope_ansirender.nim", + "subproc.nim", "awsclient.nim", "hex.c", "misc.nim", + "rope_base.nim", "switchboard.c", "box.nim", "hexdump", + "net.nim", "rope_construct.nim", "switchboard.nim", "c", + "hexdump.nim", "private", "rope_prerender.nim", + "switchboard.o", "colortable.nim", "htmlparse.nim", + "process", "rope_styles.nim", "test.c", "crownhash.nim", + "logging.nim", "progress.nim", "s3client.nim", "test.o", + "dict.nim", "macproc.c", "prp.nim", "sha.nim", + "texttable.nim", "either.nim", "macproc.nim", "pubsub.nim", + "sigv4.nim", "unicodeid.nim"] + mess1.sort() + # The padding to make up for the lost borders. + let tbl = instantTable(mess1, h3("Auto-arranged into columns")).lpad(1). + rpad(1).noBorders().topMargin(2) + + print(tbl) + + var mess2 = @[@["1, 1", "Column 2", "Column 3", "Column 4"], + @["Row 2", "Row 2", "Row 2", "Row 2"], + @["Row 3", "Row 3", "Row 3", "Row 3"], + @["Row 4", "Row 4", "Row 4", "Row 4"]] + + let t2 = quickTable(mess2, caption = h3("Table with horizontal header")). + topMargin(1).typicalBorders().colPcts([10, 40, 40, 10]) + print(t2) + + let t3 = quickTable(mess2, verticalHeaders = true, + caption = h3("Table with vertical header")) + print(t3.typicalBorders()) + + let t4 = quickTable(mess2, noheaders = true, + caption = h3("Table w/o header")). + bottomMargin(1).boldBorders().allBorders() + print(t4) + + let + st = "This is a test of something I'd really like to know about, " & + "I think??" + txt = nocolors(callout(st, 30, boxStyle = BoxStyleAscii)) + res = txt.search(text = "test") + + print(h5("Search results")) + for item in res: + var s: seq[Rune] = item.text + echo $(s) + + echo txt.debugWalk() + print(txt) + + var sometest = setWidth(callout(center(pre(txt))), 50) + #var sometest = container(callout(center(pre(txt))).lpad(10)).lpad(10) + echo txt.debugWalk() + print(center(sometest)) + + proc nestedTableTest() = + let mdText = """ +# Here's a markdown file! + +It's got some body text in it. The first paragraph really isn't +particularly long, but it is certainly quite a bit longer than the +second paragraph. So it should wrap, as long as your terminal is not +insanely wide. + +Oh look, here comes a table! + +| Example | Table | +| ------- | -------- | +| foo | bar | +| crash | override | + +## Some list +- Hello, there. +- This is an example list. +- This bullet will be long enough that it can show how we wrap bulleted text intelligently. +""" + let crazyTable = @[ + @[md(mdText), md(mdText)], + @[md(mdText), md(mdText)] + ] + let toPrint = quickTable(crazyTable, noheaders = true) + + echo toPrint.debugWalk() + print(toPrint) + + import nimutils/logging + print(h1("Testing Nimutils functionality.")) + + hexTests() + boxTest() + ulidTests() + prpTests() + shaTests() + aesGcmTests() + keyStreamTest() + dictTests() + when defined(macosx): + macProcTest() + info(em("This is a test message.")) + error(italic(underline(("So is this.")))) + nestedTableTest() + basic_subproc_tests() + instantTableTests() + print(defaultBg(fgColor("Goodbye!!", "jazzberry"))) + quit() diff --git a/nimutils.nimble b/nimutils.nimble index 635351f..8e5b7f8 100644 --- a/nimutils.nimble +++ b/nimutils.nimble @@ -4,8 +4,12 @@ version = "0.2.0" author = "John Viega" description = "Crash Ă˜verride Nim utilities" license = "Apache-2.0" +bin = @["nimutils"] # Dependencies -requires "nim >= 1.6.12" +requires "nim >= 2.0.0" requires "unicodedb == 0.12.0" + +before build: + exec thisDir() & "/bin/header_install.sh" diff --git a/nimutils/advisory_lock.nim b/nimutils/advisory_lock.nim index 1d0158d..b0e5094 100644 --- a/nimutils/advisory_lock.nim +++ b/nimutils/advisory_lock.nim @@ -1,88 +1,60 @@ ## :Author: John Viega (john@crashoverride.com) ## :Copyright: 2023, Crash Override, Inc. -import os, streams, misc, posix, file - -{.emit: """ -// Also have had this code sitting around forever and did not want to take -// the time to port it to Nim. - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -extern bool read_one(int fd, void *buf, size_t nbytes); -extern bool write_data(int fd, NCSTRING buf, NI nbytes); - -bool -lock_file(char *lfpath, int max_attempts) { - int attempt, fd, result; - pid_t pid; - - - for (attempt = 0; attempt < max_attempts; attempt++) { - if ((fd = open(lfpath, O_RDWR | O_CREAT | O_EXCL, S_IRWXU)) == -1) { - if (errno != EEXIST) { - return false; - } - if ((fd = open(lfpath, O_RDONLY)) == -1) { - return false; - } - - result = read_one(fd, &pid, sizeof(pid)); - close(fd); - if (result) { - if (pid == getpid()) { - return 1; - } - if (kill(pid, 0) == -1) { - if (errno != ESRCH) { - return false; - } - attempt--; - unlink(lfpath); - continue; - } - } - sleep(1); - continue; - } - - pid = getpid(); - if (!write_data(fd, &pid, sizeof(pid))) { - close(fd); - return false; - } - close(fd); - attempt--; - } - - /* If we've made it to here, three attempts have been made and the - * lock could not be obtained. Return an error code indicating - * failure to obtain the requested lock. - */ - return false; -} -""".} - -## This is the raw wrapping of the C func; `obtainLockFile()` accepts -## nim-native data types. -proc fLockFile*(fname: cstring, maxAttempts: cint): - bool {.importc: "lock_file".} -proc obtainLockFile*(fname: string, maxAttempts = 5): bool {.inline.} = - return fLockFile(cstring(fname), cint(maxAttempts)) +import os, file, posix, misc, subproc + + +proc flock*(fd: cint, flags: cint): cint {.discardable, importc, + header: "".} +proc fdopen*(fd: cint, mode: cstring): File {.importc, header: "".} + +type OsErrRef = ref OsError + +proc obtainLockFile*(fname: string, writeLock = false, timeout: int64 = 5000, + oflags: cint = 0): cint = + ## Obtains a lock on a file, returning the file descriptor associated + ## with it. Unlock via `unlockFd()` + var + lockflags: cint + openflags = oflags + fullpath = fname.resolvePath() + + if timeout >= 0: + lockflags = 4 + if writeLock: + lockflags = lockflags or 2 + if (openflags and 3) == 0: + openflags = openflags or 2 # Go ahead and open RDWR + else: + lockflags = lockflags or 1 + openflags = openflags and not 2 + + result = open(cstring(fullpath), openflags) + + if result == -1: + raise OsErrRef(errorCode: errno) + var + endtime: uint64 = if timeout < 0: + 0xffffffffffffffff + else: + unixTimeInMs() + uint64(timeout) + sleepdur = 16 + + while flock(result, lockflags) != 0: + if unixTimeInMs() > endTime: + raise newException(IoError, "Timeout when trying to attain file lock " & + "for " & fullpath) + sleep(sleepdur) + sleepdur = sleepdur shl 1 + +proc unlockFd*(fd: cint) = + ## Releases a file lock. + flock(fd, 8) proc writeViaLockFile*(loc: string, output: string, - release = true, - maxAttempts = 5, - ): bool = + release = true, + timeoutMs = 5000): cint = ## This uses an advisory lock to perform a read of the file, and ## then releases the lock at the end, unless release == false. ## @@ -94,28 +66,17 @@ proc writeViaLockFile*(loc: string, ## process ends. let - resolvedLoc = resolvePath(loc) - dstParts = splitPath(resolvedLoc) - lockFile = joinPath(dstParts.head, "." & dstParts.tail) - - if lockFile.obtainLockFile(maxAttempts): - try: - let f = newFileStream(resolvedLoc, fmWrite) - if f == nil: - return false - f.write(output) - f.close() - return true - finally: - if release: - try: - removeFile(lockFile) - except: - discard + fd = loc.obtainLockFile(writelock = true, timeout = timeoutMs) + + rawFdWrite(fd, cstring(output), csize_t(output.len())) + if release: + fd.unlockFd() + discard fd.close() + return 0 else: - return true + return fd -proc readViaLockFile*(loc: string, release = true, maxAttempts = 5): string = +proc readViaLockFile*(loc: string, timeoutMs = 5000): string = ## This uses an advisory lock to perform a read of the file, and ## then releases the lock at the end, unless release == false. ## @@ -128,31 +89,26 @@ proc readViaLockFile*(loc: string, release = true, maxAttempts = 5): string = ## still running and hasn't released it. let - resolvedLoc = resolvePath(loc) - dstParts = splitPath(resolvedLoc) - lockFile = joinPath(dstParts.head, "." & dstParts.tail) - - if lockFile.obtainLockFile(maxAttempts): - try: - let f = newFileStream(resolvedLoc) - if f == nil: - raise newException(ValueError, "Couldn't open file") - result = f.readAll() - f.close() - return - finally: - if release: - try: - removeFile(lockFile) - except: - discard + fd = loc.obtainLockFile(timeout = timeoutMs) + f = fdopen(fd, "r") + + result = f.readAll() + fd.unlockFd() + +proc unlock*(f: var File) = + ## Releases a file lock. + f.getFileHandle().unlockFd() + +proc lock*(f: var File, writelock = false, blocking = true) = + ## Gets a file lock from a File object. + var opts: cint = 0 + + if writelock: + opts = 2 else: - raise newException(IOError, "Couldn't obtain lock file") + opts = 1 -proc releaseLockFile*(loc: string) = - let - resolvedLoc = resolvePath(loc) - dstParts = splitPath(resolvedLoc) - lockFile = joinPath(dstParts.head, "." & dstParts.tail) + if blocking: + opts = opts and 4 - removeFile(lockFile) + flock(f.getFileHandle(), opts) diff --git a/nimutils/aes.nim b/nimutils/aes.nim index 151abe6..7509b61 100644 --- a/nimutils/aes.nim +++ b/nimutils/aes.nim @@ -6,67 +6,8 @@ const badNonceError = "GCM nonces should be exactly 12 bytes. If " & {.emit: """ // This is just a lot easier to do in straight C. -// We're going to assume headers aren't available and declare what we use. - -#include -#include -#include -#include - -#ifndef EVP_CTRL_GCM_GET_TAG -#define EVP_CIPHER_CTX void -#define EVP_CTRL_GCM_GET_TAG 0x10 -#define EVP_CTRL_GCM_SET_TAG 0x11 -#endif - -typedef void *GCM128_CONTEXT; - -extern int EVP_EncryptUpdate(void *ctx, unsigned char *out, - int *outl, const unsigned char *in, int inl); -extern int EVP_EncryptFinal(EVP_CIPHER_CTX *ctx, unsigned char *out, - int *outl); -extern int EVP_CIPHER_CTX_ctrl(EVP_CIPHER_CTX *ctx, int type, int arg, - void *ptr); -extern int EVP_EncryptInit_ex2(EVP_CIPHER_CTX *ctx, const void *type, - const unsigned char *key, const unsigned char *iv, - void *params); -extern int EVP_CipherInit_ex2(EVP_CIPHER_CTX *ctx, const void *type, - const unsigned char *key, const unsigned char *iv, - int enc, void *params); -extern int EVP_DecryptUpdate(EVP_CIPHER_CTX *ctx, unsigned char *out, - int *outl, const unsigned char *in, int inl); -extern int EVP_DecryptFinal_ex(EVP_CIPHER_CTX *ctx, unsigned char *outm, - int *outl); -extern int EVP_DecryptInit_ex(EVP_CIPHER_CTX *ctx, void *type, - void *impl, const unsigned char *key, - const unsigned char *iv); - -extern int bswap_64(int); - -typedef struct gcm_ctx { - EVP_CIPHER_CTX *aes_ctx; - int num_ops; - char *msg; - int mlen; - char *aad; - int alen; - uint8_t nonce[12]; -} gcm_ctx_t; - -typedef struct gcm_ctx_for_nonce_bump { - EVP_CIPHER_CTX *aes_ctx; - int num_ops; - char *msg; - int mlen; - char *aad; - int alen; - uint32_t highnonce; - uint64_t lownonce; -} nonce_ctx_t; - -extern char * -chex(void *ptr, unsigned int len, unsigned int start_offset, - unsigned int width); +// We assume we don't have full headers; nimugcm.h is a slimmed down version. +#include "nimugcm.h" static void bump_nonce(nonce_ctx_t *ctx) { #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ @@ -170,7 +111,6 @@ N_CDECL(int, do_gcm_decrypt)(gcm_ctx_t *ctx, void *tocast) { return 0; } - """.} {.pragma: lcrypto, cdecl, dynlib: DLLUtilName, importc.} @@ -202,7 +142,7 @@ type AesCtx* = object aesCtx: EVP_CIPHER_CTX - GcmCtx* {.importc: "gcm_ctx_t".} = object + GcmCtx* {.importc: "gcm_ctx_t", header: "nimugcm.h" .} = object aes_ctx: EVP_CIPHER_CTX num_ops: cint msg: cstring @@ -236,6 +176,7 @@ template getCipher(mode: string, key: string): EVP_CIPHER = raise newException(ValueError, "AES keys must be 16, 24 or 32 bytes.") proc initAesPRP*(ctx: var AesCtx, key: string) = + ## This sets a key for an AES context. ctx.aesCtx = EVP_CIPHER_CTX_new() let cipher = getCipher("ECB", key) @@ -243,6 +184,8 @@ proc initAesPRP*(ctx: var AesCtx, key: string) = discard EVP_EncryptInit_ex(ctx.aesCtx, cipher, nil, cstring(key), nil) proc aesPrp*(ctx: AesCtx, input: string): string = + ## This is for using AES a a basic PRP (pseudo-random permutation). + ## Only use this interface if you know what you're doing. var i: cint if len(input) != 16: raise newException(ValueError, "The AES PRP operates on 16 byte strings.") @@ -252,6 +195,10 @@ proc aesPrp*(ctx: AesCtx, input: string): string = cint(16)) proc aesBrb*(ctx: AesCtx, input: string): string = + ## This is for using AES a a basic PRP (pseudo-random permutation). + ## Only use this interface if you know what you're doing. + ## + ## Specifically, this is the inverse of the primary permutation. var i: cint if len(input) != 16: raise newException(ValueError, "The AES BRB operates on 16 byte strings.") @@ -262,6 +209,11 @@ proc aesBrb*(ctx: AesCtx, input: string): string = proc gcmInitEncrypt*(ctx: var GcmCtx, key: string, nonce = ""): string {.discardable} = + ## Initialize authenticated encryption using AES-GCM. Nonces are + ## (intentionally) constrained to always be 12 bytes, and if you do + ## not pass in a nonce, you will get a random value. + ## + ## The nonce used is always returned. ctx.aesCtx = EVP_CIPHER_CTX_new() let cipher = getCipher("GCM", key) @@ -283,10 +235,18 @@ proc gmacInit*(ctx: var GcmCtx, key: string, nonce = ""): return gcmInitEncrypt(ctx, key, nonce) proc aesPrfOneShot*(key: string, outlen: int, start: string = ""): string = + ## This runs AES as a pseudo-random function with a fixed-size (16 + ## byte) input yielding an output of the length specified (in + ## bytes). + ## + ## The `start` parameter is essentially the nonce; do not reuse it. + ## + ## This is an `expert mode` interface. + var ctx: AesCtx nonce: pointer = nil - outbuf: ptr char = cast[ptr char](alloc(outlen)) + outbuf: pointer = alloc(outlen) ctx.aesCtx = EVP_CIPHER_CTX_new() @@ -307,6 +267,14 @@ proc aesPrfOneShot*(key: string, outlen: int, start: string = ""): string = proc aesCtrInPlaceOneshot*(key: string, instr: pointer, l: cint, start: string = "") = + ## This also is an `expert mode` interface, don't use counter mode + ## unless you know exactly what you're doing. GCM mode is more + ## appropriate. + ## + ## This runs counter mode, modifying a buffer in-place. + ## + ## The final parameter is a nonce. + var ctx: AesCtx nonce: pointer = nil @@ -326,9 +294,17 @@ proc aesCtrInPlaceOneshot*(key: string, instr: pointer, l: cint, raise newException(IoError, "Could not generate keystream") proc aesCtrInPlaceOneshot*(key, instr: string, start: string = "") = + ## This also is an `expert mode` interface, don't use counter mode + ## unless you know exactly what you're doing. GCM mode is more + ## appropriate. + ## + ## This runs counter mode, modifying a buffer in-place. + ## + ## The final parameter is a nonce. aesCtrInPlaceOneshot(key, addr instr[0], cint(instr.len()), start) proc gcmInitDecrypt*(ctx: var GcmCtx, key: string) = + ## Initializes the decryption side of GCM. ctx.aesCtx = EVP_CIPHER_CTX_new() let cipher = getCipher("GCM", key) @@ -336,6 +312,8 @@ proc gcmInitDecrypt*(ctx: var GcmCtx, key: string) = EVP_DecryptInit_ex2(ctx.aesCtx, cipher, cstring(key), nil, nil) proc gcmEncrypt*(ctx: var GcmCtx, msg: string, aad = ""): string = + ## GCM-encrypts a single message in a session, using the + ## state setup by gcmEncryptInit() var outbuf: ptr char = cast[ptr char](alloc(len(msg) + 16)) ctx.aad = cstring(aad) @@ -350,17 +328,27 @@ proc gcmEncrypt*(ctx: var GcmCtx, msg: string, aad = ""): string = dealloc(outbuf) proc gmac*(ctx: var GcmCtx, msg: string): string = + ## Runs the GMAC message authentication code algorithm on a single + ## message, for a session set up via gcmInitEncrypt(). This is the + ## same as passing a null message, but providing additional data to + ## authenticate. + ## + ## The receiver should always use gcmDecrypt() to validate. return gcmEncrypt(ctx, msg = "", aad = msg) proc gcmGetNonce*(ctx: var GcmCtx): string = + ## Returns the sessions nonce. for i, ch in ctx.nonce: result.add(char(ctx.nonce[i])) proc gmacGetNonce*(ctx: var GcmCtx): string = + ## Returns the sessions nonce. return gcmGetNonce(ctx) proc gcmDecrypt*(ctx: var GcmCtx, msg: string, nonce: string, aad = ""): Option[string] = + ## Performs validation and decryption of an encrypted message for a session + ## initialized via `gcmInitDecrypt()` if len(msg) < 16: raise newException(ValueError, "Invalid GCM Ciphertext (too short)") @@ -383,48 +371,3 @@ proc gcmDecrypt*(ctx: var GcmCtx, msg: string, nonce: string, result = some(bytesToString(outbuf, len(msg) - 16)) dealloc(outbuf) - -when isMainModule: - import strutils, hexdump - - var - encCtx: GcmCtx - decCtx: GcmCtx - nonce: string - ct: string - pt = "This is a test between disco and death" - key = "0123456789abcdef" - gcmInitEncrypt(encCtx, key) - gcmInitDecrypt(decCtx, key) - - echo "Initial pt: ", pt - - for i in 1 .. 3: - ct = encCtx.gcmEncrypt(pt) - nonce = encCtx.gcmGetNonce() - pt = decCtx.gcmDecrypt(ct, nonce).get("") - - echo "Nonce: ", nonce.toHex().toLowerAscii() - echo "CT: " - echo strDump(ct) - echo "Decrypted: ", pt - - echo "Keystream test: " - - let - stream1 = aesPrfOneShot(key, 200) - stream2 = aesPrfOneShot(key, 200) - - echo stream1.toHex() - assert len(stream1) == 200 - assert stream1 == stream2 - - let text = "This is a test, yo, dawg" - - aesCtrInPlaceOneshot(key, text) - - echo "Covered dog: ", text.hex() - - aesCtrInPlaceOneshot(key, text) - - echo text diff --git a/nimutils/awsclient.nim b/nimutils/awsclient.nim index b20e1ea..c6ab916 100644 --- a/nimutils/awsclient.nim +++ b/nimutils/awsclient.nim @@ -1,3 +1,8 @@ +## Hopefully this will die before too long; I intend to instead wrap a +## library from AWS. +## +## This is some third-party thing that (barely) crosses the line into +## acceptable. #[ # AwsClient diff --git a/nimutils/box.nim b/nimutils/box.nim index 7a1b51e..4463710 100644 --- a/nimutils/box.nim +++ b/nimutils/box.nim @@ -45,6 +45,15 @@ ## I did just learn that you can do [: instead to get UFCS semantics. ## But when you get it wrong, the error messages are not even a little ## helpful. +## +## ******************************************************************** +## * NOTE that I expect to update this for the sake of Con4m, with a * +## * version that will better support reference semantics, optionally * +## * carry around type information, and will be able to hold a much * +## * broader set of Nim objects. This interface might get preserved * +## * as-is, might get renamed, or might get full deprecated... * +## * so beware! * +## ******************************************************************** import std/typetraits import std/hashes @@ -82,7 +91,6 @@ proc arrItemType*[T](a: openarray[T]): auto = return default(T) proc arrItemType*(a: BoxAtom): BoxAtom = a - proc `==`*(box1, box2: Box) : bool {.noSideEffect.} = # The noSideEffect works around a bug in nim2.0 if box1.kind != box2.kind: @@ -232,6 +240,7 @@ proc pack*[T](x: T): Box = raise newException(ValueError, "Bad type to pack: " & $(T.type)) proc `$`*(x: Box): string = + ## Produces a basic string representation of a box. case x.kind of MkFloat: return $(x.f) @@ -260,6 +269,7 @@ proc `$`*(x: Box): string = return "" proc boxToJson*(b: Box): string = + ## Produces a basic string representation of a box, as a JSON object. var addComma: bool = false case b.kind @@ -281,67 +291,3 @@ proc boxToJson*(b: Box): string = result = result & " }" else: return "null" # Boxed objects not supported - -when isMainModule: - var - i1 = "a" - l1 = @["a", "b", "c"] - l2 = @["d", "e", "f"] - l3 = @["g", "h", "i"] - l123 = @[l1, l2, l3] - b1, b123: Box - o123: seq[seq[string]] = @[] - oMy: seq[Box] = @[] - a1 = pack(i1) - - echo typeof(a1) - echo unpack[string](a1) - b1 = pack(l1) - echo b1 - echo unpack[seq[string]](b1) - b123 = pack(l123) - echo b123 - echo typeof(b123) - echo typeof(o123) - o123 = unpack[seq[seq[string]]](b123) - echo o123 - oMy = unpack[seq[Box]](b123) - echo oMy - - var myDict = newTable[string, seq[string]]() - - myDict["foo"] = @["a", "b"] - myDict["bar"] = @["b"] - myDict["boz"] = @["c"] - myDict["you"] = @["d"] - - import streams - - let - f = newFileStream("box.nim", fmRead) - contents = f.readAll()[0 .. 20] - - myDict["file"] = @[contents] - - let - dictBox = pack(myDict) - listbox = pack(l1) - - var outlist: l1.type - unpack(listbox, outlist) - - echo "Here's the listbox: ", listbox - echo "Here it is unpacked: ", outlist - - var newDict: TableRef[string, seq[string]] - - unpack(dictBox, newDict) - - echo "Here's the dictbox(nothing should be quoted): ", dictBox - echo "Here it is unpacked (should have quotes): ", newDict - echo "Here it is, boxed, as Json: ", boxToJson(dictBox) - - # This shouldn't work w/o a custom handler. - # import sugar - # var v: ()->int - # unpack[()->int](b123, v) diff --git a/nimutils/c/crownhash.h b/nimutils/c/crownhash.h new file mode 100644 index 0000000..e138a59 --- /dev/null +++ b/nimutils/c/crownhash.h @@ -0,0 +1,111 @@ +#ifndef __CROWNHASH_H__ +#define __CROWNHASH_H__ +#include +#include +#include +#include +#include + +#ifndef HATRACK_THREADS_MAX +#define HATRACK_THREADS_MAX 4096 +#endif + +#if HATRACK_THREADS_MAX > 32768 +#error "Vector assumes HATRACK_THREADS_MAX is no higher than 32768" +#endif + +typedef void (*mmm_cleanup_func)(void *, void *); + +typedef struct mmm_header_st mmm_header_t; +typedef struct mmm_free_tids_st mmm_free_tids_t; + +extern __thread int64_t mmm_mytid; +extern __thread pthread_once_t mmm_inited; +extern _Atomic uint64_t mmm_epoch; +extern uint64_t mmm_reservations[HATRACK_THREADS_MAX]; + +struct mmm_header_st { + alignas(16) + mmm_header_t *next; + _Atomic uint64_t create_epoch; + _Atomic uint64_t write_epoch; + uint64_t retire_epoch; + mmm_cleanup_func cleanup; + void *cleanup_aux; // Data needed for cleanup, usually the object + alignas(16) + uint8_t data[]; +}; + +struct mmm_free_tids_st { + mmm_free_tids_t *next; + uint64_t tid; +}; + +typedef struct { + uint64_t w1; + uint64_t w2; +} hatrack_hash_t; + +typedef struct { + void *item; + uint64_t info; +} crown_record_t; + +typedef struct { + _Atomic hatrack_hash_t hv; + _Atomic crown_record_t record; + _Atomic uint64_t neighbor_map; + +} crown_bucket_t; + +typedef struct crown_store_st crown_store_t; + +struct crown_store_st { + alignas(8) + uint64_t last_slot; + uint64_t threshold; + _Atomic uint64_t used_count; + _Atomic(crown_store_t *) store_next; + _Atomic bool claimed; + alignas(16) + crown_bucket_t buckets[]; +}; + +typedef struct { + alignas(8) + _Atomic(crown_store_t *) store_current; + _Atomic uint64_t item_count; + _Atomic uint64_t help_needed; + uint64_t next_epoch; +} crown_t; + +typedef struct { + void *key; + void *value; +} hatrack_dict_item_t; + + +typedef struct hatrack_dict_st hatrack_dict_t; + +typedef void *hatrack_dict_key_t; +typedef void *hatrack_dict_value_t; + +typedef hatrack_hash_t (*hatrack_hash_func_t)(void *); +typedef void (*hatrack_mem_hook_t)(void *, void *); + +typedef union { + int64_t offset_info; + hatrack_hash_func_t custom_hash; +} hatrack_hash_info_t; + +struct hatrack_dict_st { + crown_t crown_instance; + hatrack_hash_info_t hash_info; + hatrack_mem_hook_t free_handler; + hatrack_mem_hook_t key_return_hook; + hatrack_mem_hook_t val_return_hook; + uint32_t key_type; + bool slow_views; + bool sorted_views; +}; +#endif diff --git a/nimutils/headers/gumbo.nim b/nimutils/c/gumbo.h similarity index 99% rename from nimutils/headers/gumbo.nim rename to nimutils/c/gumbo.h index 1630bdd..a4c610e 100644 --- a/nimutils/headers/gumbo.nim +++ b/nimutils/c/gumbo.h @@ -1,4 +1,3 @@ -{.emit: """ // Copyright 2010 Google Inc. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -968,4 +967,3 @@ const char *gumbo_status_to_string(GumboOutputStatus status); #endif #endif // GUMBO_GUMBO_H_ -""".} diff --git a/nimutils/hex.c b/nimutils/c/hex.c similarity index 100% rename from nimutils/hex.c rename to nimutils/c/hex.c diff --git a/nimutils/hex.h b/nimutils/c/hex.h similarity index 100% rename from nimutils/hex.h rename to nimutils/c/hex.h diff --git a/nimutils/c/macproc.c b/nimutils/c/macproc.c new file mode 100644 index 0000000..954c2b9 --- /dev/null +++ b/nimutils/c/macproc.c @@ -0,0 +1,283 @@ +#include "macproc.h" + +int maxalloc; + +void +del_procinfo(procinfo_t *cur) { + int i = 0; + + if (cur == NULL) { + return; + } + + while (cur[i].pid) { + if (cur[i].username != NULL) { + free(cur[i].username); + } + if (cur[i].path != NULL) { + free(cur[i].path); + } + free(cur[i].memblock); + + if (cur[i].numgroups != 0) { + for (int j = 0; j < cur[i].numgroups; j++) { + free(cur[i].gids[j].name); + } + free(cur[i].gids); + } + i++; + } + free(cur); +} + +void __attribute__((constructor)) set_maxalloc() { + int kargmax[] = {CTL_KERN, KERN_ARGMAX}; + size_t size; + + if (sysctl(kargmax, 2, &maxalloc, &size, NULL, 0) == -1) { + abort(); + } +} + +void +get_more_procinfo(procinfo_t *cur, int pid) { + char *path = calloc(PROC_PIDPATHINFO_MAXSIZE, 1); + + proc_pidpath(pid, path, PROC_PIDPATHINFO_MAXSIZE); + cur->path = realloc(path, strlen(path) + 1); + + + int procargs[] = {CTL_KERN, KERN_PROCARGS2, pid}; + + size_t size; + + char *memblock = (char *)calloc(maxalloc, 1); + + if (!memblock) { + return; + } + + size = maxalloc; + + if (sysctl(procargs, 3, memblock, &size, NULL, 0) == -1) { + free(memblock); + cur->argc = 0; + cur->envc = 0; + return; + } + + memblock = realloc(memblock, size); + cur->argc = *(int *)memblock; + cur->memblock = memblock; + + char *p = memblock + sizeof(int); + + // Skip path info; it's only partial, which is why we use proc_pidpath() + while(*p != 0) { p++; } + + // Skip any nulls after the path; + while(*p == 0) { p++; } + + cur->argv = calloc(sizeof(char *), cur->argc); + + for (int i = 0; i < cur->argc; i++) { + cur->argv[i] = p; + + while (*p) p++; + p++; + } + + char *env_start = p; + + cur->envc = 0; + + while (*p) { + cur->envc++; + while(*p++); + } + + p = env_start; + cur->envp = calloc(sizeof(char *), cur->envc); + + for (int i = 0; i < cur->envc; i++) { + cur->envp[i] = p; + + while (*p) p++; + } +} + +/* Even though this seems like it allocates an insane amount of memory, + * It's still plenty fast. + * + * For instance, I get a len of 284472 (which I expect is the next pid?) + * but there are only 438 procs. + * + * The OS seems to put valid results all together, so the break + * statement appears to work fine. + * + * But I've tested performance w/ a continue in the second loop + * instead, and it's definitely a lot slower, but still runs in less + * than .1 sec on my laptop (as opposed to around .01 seconds). + */ +procinfo_t * +proc_list(size_t *count) { + int err; + struct kinfo_proc *result, *to_free; + procinfo_t *pi; + procinfo_t *to_return; + int name[] = { CTL_KERN, KERN_PROC, KERN_PROC_ALL}; + size_t i, len; + size_t valid = 0; + int failsafe = 0; + + // This loop should only ever run once from what I can tell, because + // the OS has us massively over-allocate. + // If this goes more than 10 loops, we bail. + while (true) { + err = sysctl(name, 3, NULL, &len, NULL, 0); + if (err != 0) { + return NULL; + } + + result = (struct kinfo_proc *)calloc(sizeof(struct kinfo_proc), len); + to_free = result; + + if (result == NULL) { + return NULL; + } + if (sysctl(name, 3, result, &len, NULL, 0) == -1) { + free(result); + if (failsafe++ == 10) { + return NULL; + } + } + else { + break; + } + } + + // Add an extra one where we drop in pid = 0 as a sentinel. + // Not that we're likely to need it. + pi = (procinfo_t *)calloc(sizeof(procinfo_t), len + 1); + to_return = pi; + + for (i = 0; i < len; i++) { + int pid = result->kp_proc.p_pid; + + if (!pid) { + pi->pid = 0; + break; + } + + valid = valid + 1; + + pi->pid = pid; + pi->ppid = result->kp_eproc.e_ppid; + pi->uid = result->kp_eproc.e_pcred.p_ruid; + pi->gid = result->kp_eproc.e_pcred.p_rgid; + pi->euid = result->kp_eproc.e_ucred.cr_uid; + pi->numgroups = result->kp_eproc.e_ucred.cr_ngroups; + + struct passwd *pwent = getpwuid(pi->uid); + pi->username = strdup(pwent->pw_name); + + struct group *ginfo; + + if (pi->numgroups == 0) { + pi->gids = NULL; + } else { + // Seems to be a ton of dupes, so skipping them. + int sofar = 0; + pi->gids = calloc(sizeof(gidinfo_t), pi->numgroups); + + for (int i = 0; i < pi->numgroups; i++) { + for (int j = 0; j < sofar; j++) { + if(pi->gids[j].id == + result->kp_eproc.e_ucred.cr_groups[i]) { + goto skip_copy; + } + } + pi->gids[sofar].id = result->kp_eproc.e_ucred.cr_groups[i]; + ginfo = getgrgid(pi->gids[i].id); + pi->gids[sofar].name = strdup(ginfo->gr_name); + sofar++; + + skip_copy: + continue; + } + pi->numgroups = sofar; + } + + get_more_procinfo(pi, pid); + + pi++; + result++; + } + + free(to_free); + + *count = valid; + + to_return[valid].pid = 0; + + return realloc(to_return, sizeof(procinfo_t) * (valid + 1)); +} + +procinfo_t * +proc_list_one(size_t *count, int pid) { + int err; + struct kinfo_proc *result; + procinfo_t *to_return; + int name[] = { CTL_KERN, KERN_PROC, KERN_PROC_PID, pid}; + size_t i, len; + size_t valid = 0; + int failsafe = 0; + + while (true) { + err = sysctl(name, 4, NULL, &len, NULL, 0); + if (err != 0) { + return NULL; + } + + result = (struct kinfo_proc *)calloc(sizeof(struct kinfo_proc), len); + if (result == NULL) { + return NULL; + } + if (sysctl(name, 4, result, &len, NULL, 0) == -1) { + free(result); + + if (failsafe++ == 10) { + return NULL; + } + } + else { + if (len != 1) { + return NULL; + } + break; + } + } + + to_return = (procinfo_t *)calloc(sizeof(procinfo_t), len); + + for (i = 0; i < len; i++) { + struct kinfo_proc *oneProc = &result[i]; + int pid = oneProc->kp_proc.p_pid; + + if (!pid) continue; + + valid = valid + 1; + + to_return[i].pid = pid; + to_return[i].ppid = oneProc->kp_eproc.e_ppid; + to_return[i].uid = oneProc->kp_eproc.e_ucred.cr_uid; + + get_more_procinfo(&to_return[i], pid); + } + + free(result); + + *count = valid; + + return to_return; +} diff --git a/nimutils/c/macproc.h b/nimutils/c/macproc.h new file mode 100644 index 0000000..226263e --- /dev/null +++ b/nimutils/c/macproc.h @@ -0,0 +1,47 @@ +#ifndef __MACPROC_H__ +#define __MACPROC_H__ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +typedef struct { + int id; + char *name; +} gidinfo_t; + +typedef struct { + int pid; + int uid; + int gid; + int euid; + int ppid; + char *username; + char *path; + int argc; + int envc; + char *memblock; + char **argv; + char **envp; + int numgroups; + gidinfo_t *gids; +} procinfo_t; + +extern procinfo_t *proc_list(size_t *count); +extern procinfo_t *proc_list_one(size_t *count, int pid); +extern void del_procinfo(procinfo_t *cur); + +extern int errno; +extern void del_procinfo(procinfo_t *); +extern void get_more_procinfo(procinfo_t *, int); +extern procinfo_t *proc_list(size_t *); +extern procinfo_t *proc_list_one(size_t *, int); +#endif diff --git a/nimutils/headers/md4c.nim b/nimutils/c/md4c.h similarity index 99% rename from nimutils/headers/md4c.nim rename to nimutils/c/md4c.h index a4befff..80eac1c 100644 --- a/nimutils/headers/md4c.nim +++ b/nimutils/c/md4c.h @@ -1,4 +1,3 @@ -{.emit: """ /* * MD4C: Markdown parser for C * (http://github.com/mity/md4c) @@ -9870,7 +9869,7 @@ entity_lookup(const char* name, size_t name_size) // John's stuff. typedef void (*CB_TYPE)(const char *, unsigned int, void*); -N_LIB_PRIVATE N_CDECL(void, nimu_process_markdown)(NCSTRING s_p0, unsigned int n_p1, void* p_p2); +N_LIB_PRIVATE N_CDECL(void, nimu_process_markdown)(NU8* s_p0, unsigned int n_p1, void* p_p2); MD_HTML_CALLBACKS x = { .process_output = nimu_process_markdown, @@ -9889,4 +9888,3 @@ c_markdown_to_html(char *mdoc, size_t len, void *outobj, size_t flags) { #endif #endif /* MD4C_H */ -""".} diff --git a/nimutils/c/nimugcm.h b/nimutils/c/nimugcm.h new file mode 100644 index 0000000..5fa3b3c --- /dev/null +++ b/nimutils/c/nimugcm.h @@ -0,0 +1,60 @@ +#ifndef __NIMU_GCM_H__ +#define __NIMU_GCM_H__ + +#include +#include +#include +#include + +#ifndef EVP_CTRL_GCM_GET_TAG +#define EVP_CIPHER_CTX void +#define EVP_CTRL_GCM_GET_TAG 0x10 +#define EVP_CTRL_GCM_SET_TAG 0x11 +#endif + +typedef void *GCM128_CONTEXT; + +extern int EVP_EncryptUpdate(void *ctx, unsigned char *out, + int *outl, const unsigned char *in, int inl); +extern int EVP_EncryptFinal(EVP_CIPHER_CTX *ctx, unsigned char *out, + int *outl); +extern int EVP_CIPHER_CTX_ctrl(EVP_CIPHER_CTX *ctx, int type, int arg, + void *ptr); +extern int EVP_EncryptInit_ex2(EVP_CIPHER_CTX *ctx, const void *type, + const unsigned char *key, const unsigned char *iv, + void *params); +extern int EVP_CipherInit_ex2(EVP_CIPHER_CTX *ctx, const void *type, + const unsigned char *key, const unsigned char *iv, + int enc, void *params); +extern int EVP_DecryptUpdate(EVP_CIPHER_CTX *ctx, unsigned char *out, + int *outl, const unsigned char *in, int inl); +extern int EVP_DecryptFinal_ex(EVP_CIPHER_CTX *ctx, unsigned char *outm, + int *outl); +extern int EVP_DecryptInit_ex(EVP_CIPHER_CTX *ctx, void *type, + void *impl, const unsigned char *key, + const unsigned char *iv); + +extern int bswap_64(int); + +typedef struct gcm_ctx { + EVP_CIPHER_CTX *aes_ctx; + int num_ops; + char *msg; + int mlen; + char *aad; + int alen; + uint8_t nonce[12]; +} gcm_ctx_t; + +typedef struct gcm_ctx_for_nonce_bump { + EVP_CIPHER_CTX *aes_ctx; + int num_ops; + char *msg; + int mlen; + char *aad; + int alen; + uint32_t highnonce; + uint64_t lownonce; +} nonce_ctx_t; + +#endif diff --git a/nimutils/subproc.c b/nimutils/c/subproc.c similarity index 86% rename from nimutils/subproc.c rename to nimutils/c/subproc.c index f6770f8..8794795 100644 --- a/nimutils/subproc.c +++ b/nimutils/c/subproc.c @@ -81,7 +81,7 @@ subproc_pass_to_stdin(subprocess_t *ctx, char *str, size_t len, bool close_fd) return false; } - sb_init_party_input_buf(&ctx->sb, &ctx->str_stdin, str, len, false, + sb_init_party_input_buf(&ctx->sb, &ctx->str_stdin, str, len, true, true, close_fd); if (ctx->run) { @@ -125,7 +125,6 @@ subproc_set_passthrough(subprocess_t *ctx, unsigned char which, bool combine) return true; } - /* * This controls whether input from a file descriptor is captured into * a string that is available when the process ends. @@ -232,6 +231,96 @@ subproc_get_pty_fd(subprocess_t *ctx) return ctx->pty_fd; } +void +pause_passthrough(subprocess_t *ctx, unsigned char which) +{ + /* + * Since there's no real consequence to trying to pause a + * subscription that doesn't exist, we'll just try to pause every + * subscription implied by `which`. Note that if we see stderr, we + * try to unsubscribe it from both the parent's stdout and the + * parent's stderr; no strong need to care whether they were + * combined or not here.. + */ + + if (which & SP_IO_STDIN) { + if (ctx->pty_fd) { + sb_pause_route(&ctx->sb, &ctx->parent_stdin, &ctx->subproc_stdout); + } else { + sb_pause_route(&ctx->sb, &ctx->parent_stdin, &ctx->subproc_stdin); + } + } + if (which & SP_IO_STDOUT) { + sb_pause_route(&ctx->sb, &ctx->subproc_stdout, &ctx->parent_stdout); + } + if (!ctx->pty_fd && (which & SP_IO_STDERR)) { + sb_pause_route(&ctx->sb, &ctx->subproc_stderr, &ctx->parent_stdout); + sb_pause_route(&ctx->sb, &ctx->subproc_stderr, &ctx->parent_stderr); + } +} + +void +resume_passthrough(subprocess_t *ctx, unsigned char which) +{ + /* + * Since there's no real consequence to trying to pause a + * subscription that doesn't exist, we'll just try to pause every + * subscription implied by `which`. Note that if we see stderr, we + * try to unsubscribe it from both the parent's stdout and the + * parent's stderr; no strong need to care whether they were + * combined or not here.. + */ + + if (which & SP_IO_STDIN) { + if (ctx->pty_fd) { + sb_resume_route(&ctx->sb, &ctx->parent_stdin, &ctx->subproc_stdout); + } else { + sb_resume_route(&ctx->sb, &ctx->parent_stdin, &ctx->subproc_stdin); + } + } + if (which & SP_IO_STDOUT) { + sb_resume_route(&ctx->sb, &ctx->subproc_stdout, &ctx->parent_stdout); + } + if (!ctx->pty_fd && (which & SP_IO_STDERR)) { + sb_resume_route(&ctx->sb, &ctx->subproc_stderr, &ctx->parent_stdout); + sb_resume_route(&ctx->sb, &ctx->subproc_stderr, &ctx->parent_stderr); + } +} + +void +pause_capture(subprocess_t *ctx, unsigned char which) +{ + if (which & SP_IO_STDIN) { + sb_pause_route(&ctx->sb, &ctx->parent_stdin, &ctx->capture_stdin); + } + + if (which & SP_IO_STDOUT) { + sb_pause_route(&ctx->sb, &ctx->subproc_stdout, &ctx->capture_stdout); + } + + if ((which & SP_IO_STDERR) && !ctx->pty_fd) { + sb_pause_route(&ctx->sb, &ctx->subproc_stderr, &ctx->capture_stdout); + sb_pause_route(&ctx->sb, &ctx->subproc_stderr, &ctx->capture_stderr); + } +} + +void +resume_capture(subprocess_t *ctx, unsigned char which) +{ + if (which & SP_IO_STDIN) { + sb_resume_route(&ctx->sb, &ctx->parent_stdin, &ctx->capture_stdin); + } + + if (which & SP_IO_STDOUT) { + sb_resume_route(&ctx->sb, &ctx->subproc_stdout, &ctx->capture_stdout); + } + + if ((which & SP_IO_STDERR) && !ctx->pty_fd) { + sb_resume_route(&ctx->sb, &ctx->subproc_stderr, &ctx->capture_stdout); + sb_resume_route(&ctx->sb, &ctx->subproc_stderr, &ctx->capture_stderr); + } +} + static void setup_subscriptions(subprocess_t *ctx, bool pty) { @@ -521,8 +610,7 @@ termcap_set(struct termios *termcap) { * sufficient calls to poll for IP, instead of having it run to * completion. * - * If you use this, call subproc_poll() until it returns false, - * at which point, call subproc_prepare_results(). + * If you use this, call subproc_poll() until it returns false */ void subproc_start(subprocess_t *ctx) @@ -546,20 +634,6 @@ subproc_poll(subprocess_t *ctx) return sb_operate_switchboard(&ctx->sb, false); } -/* - * Call this before querying any results. - */ -void -subproc_prepare_results(subprocess_t *ctx) -{ - sb_prepare_results(&ctx->sb); - - // Post-run cleanup. - if (ctx->use_pty) { - tcsetattr(0, TCSANOW, &ctx->saved_termcap); - } -} - /* * Spawns a process, and runs it until the process has ended. The * process must first be set up with `subproc_init()` and you may @@ -572,10 +646,16 @@ subproc_run(subprocess_t *ctx) { subproc_start(ctx); sb_operate_switchboard(&ctx->sb, true); - - subproc_prepare_results(ctx); } + +void +subproc_reset_terminal(subprocess_t *ctx) { + // Post-run cleanup. + if (ctx->use_pty) { + tcsetattr(0, TCSANOW, &ctx->saved_termcap); + } +} /* * This destroys any allocated memory inside a `subproc` object. You * should *not* call this until you're done with the `sb_result_t` @@ -589,8 +669,9 @@ subproc_run(subprocess_t *ctx) void subproc_close(subprocess_t *ctx) { + subproc_reset_terminal(ctx); sb_destroy(&ctx->sb, false); - + deferred_cb_t *cbs = ctx->deferred_cbs; deferred_cb_t *next; @@ -642,7 +723,8 @@ sp_result_capture(sp_result_t *ctx, char *tag, size_t *outlen) char * subproc_get_capture(subprocess_t *ctx, char *tag, size_t *outlen) { - return sp_result_capture(&ctx->sb.result, tag, outlen); + sb_get_results(&ctx->sb, &ctx->result); + return sp_result_capture(&ctx->result, tag, outlen); } int @@ -708,6 +790,7 @@ subproc_get_extra(subprocess_t *ctx) return sb_get_extra(&ctx->sb); } + #ifdef SB_TEST void capture_tty_data(switchboard_t *sb, party_t *party, char *data, size_t len) diff --git a/nimutils/switchboard.c b/nimutils/c/switchboard.c similarity index 83% rename from nimutils/switchboard.c rename to nimutils/c/switchboard.c index a6ce7c8..a4a20c6 100644 --- a/nimutils/switchboard.c +++ b/nimutils/c/switchboard.c @@ -153,9 +153,9 @@ register_loner(switchboard_t *ctx, party_t *loner) * pass in a sockfd that is already open, and has already had listen() * called on it. * - * We force this into non-blocking mode; when there's something ready - * to accept(), we'll get triggered that a read is available. We - * then accept() for you, and pass that to a callback. + * When there's something ready to accept(), we'll get triggered that + * a read is available. We then accept() for you, and pass that to a + * callback. * * You can then register the fd connection in the same switchboard if * you like. It's not done for you at this level, though. @@ -184,9 +184,6 @@ sb_init_party_listener(switchboard_t *ctx, party_t *party, int sockfd, lobj->accept_cb = (accept_cb_decl)callback; lobj->saved_flags = fcntl(sockfd, F_GETFL, 0); - int flags = lobj->saved_flags | O_NONBLOCK; - - fcntl(sockfd, F_SETFL, flags); register_read_fd(ctx, party); if (stop_when_closed) { @@ -271,8 +268,15 @@ sb_new_party_fd(switchboard_t *ctx, int fd, int perms, */ void sb_init_party_input_buf(switchboard_t *ctx, party_t *party, char *input, - size_t len, bool free, bool close_fd_when_done) + size_t len, bool dup, bool free, bool close_fd_when_done) { + char *to_set = input; + + if (dup) { + free = true; + to_set = (char *)calloc(len + 1, 0); + memcpy(to_set, dup, len); + } party->open_for_read = true; party->open_for_write = false; party->can_read_from_it = true; @@ -280,7 +284,7 @@ sb_init_party_input_buf(switchboard_t *ctx, party_t *party, char *input, party->party_type = PT_STRING; str_src_party_t *sobj = get_sstr_obj(party); - sobj->strbuf = input; + sobj->strbuf = to_set; sobj->len = len; sobj->free_on_close = free; sobj->close_fd_when_done = close_fd_when_done; @@ -289,15 +293,39 @@ sb_init_party_input_buf(switchboard_t *ctx, party_t *party, char *input, } party_t * -sb_new_party_input_buf(switchboard_t *ctx, char *input, size_t len, bool free, - bool close_fd_when_done) +sb_new_party_input_buf(switchboard_t *ctx, char *input, size_t len, + bool dup, bool free, bool close_fd_when_done) { party_t *result = (party_t *)calloc(sizeof(party_t), 1); - sb_init_party_input_buf(ctx, result, input, len, free, close_fd_when_done); + sb_init_party_input_buf(ctx, result, input, len, dup, free, + close_fd_when_done); return result; } +void +sb_party_input_buf_new_string(party_t *party, char *input, size_t len, + bool dup, bool close_fd_when_done) +{ + if (party->party_type != PT_STRING || !party->can_read_from_it) { + return; + } + str_src_party_t *sobj = get_sstr_obj(party); + if (sobj->free_on_close && sobj->strbuf) { + free(sobj->strbuf); + } + sobj->len = len; + sobj->close_fd_when_done = close_fd_when_done; + + if (dup) { + sobj->strbuf = (char *)calloc(len + 1, 1); + memcpy(sobj->strbuf, input, len); + sobj->free_on_close = true; + } else { + sobj->strbuf = input; + } +} + /* * When you want to capture process output, but don't need it until * the process is closes, this is your huckleberry. You can use one of @@ -558,7 +586,6 @@ sb_route(switchboard_t *ctx, party_t *read_from, party_t *write_to) return false; } - if (read_from->party_type == PT_STRING) { if (write_to->party_type != PT_FD) { return false; @@ -618,10 +645,156 @@ sb_route(switchboard_t *ctx, party_t *read_from, party_t *write_to) r_fd_obj->subscribers = subscription; } - return true; } +/* + * Pause the specified routing (unsubscribe), if one is active. + * Returns `true` if the subscription was marked as paused. + * + * This does not consider whether the fds are closed. + * + * Currently, there's no explicit facility for removing subscriptions. + * Just pause it and never unpause it! + */ +bool +sb_pause_route(switchboard_t *ctx, party_t *read_from, party_t *write_to) +{ + if (read_from == NULL || write_to == NULL) { + return false; + } + + fd_party_t *reader = get_fd_obj(read_from); + subscription_t *cur = reader->subscribers; + + while (cur != NULL) { + if (cur->subscriber != write_to) { + cur = cur->next; + continue; + } + if (cur->paused) { + return false; + } else { + cur->paused = true; + return true; + } + } + return false; +} + +/* + * Resumes the specified subscription. Returns `true` if resumption + * was successful, and `false` if not, including cases where the + * subscription was already active.. + */ +bool +sb_resume_route(switchboard_t *ctx, party_t *read_from, party_t *write_to) +{ + if (read_from == NULL || write_to == NULL) { + return false; + } + + fd_party_t *reader = get_fd_obj(read_from); + subscription_t *cur = reader->subscribers; + + while (cur != NULL) { + if (cur->subscriber != write_to) { + cur = cur->next; + continue; + } + if (cur->paused) { + cur->paused = true; + return true; + } else { + return false; + } + } + return false; +} + +/* + * Returns true if the subscription is active, and in an unpaused state. + */ +bool +sb_route_is_active(switchboard_t *ctx, party_t *read_from, party_t *write_to) +{ + if (read_from == NULL || write_to == NULL) { + return false; + } + + if (!read_from->open_for_read || !write_to->open_for_write) { + return false; + } + + fd_party_t *reader = get_fd_obj(read_from); + subscription_t *cur = reader->subscribers; + + while (cur != NULL) { + if (cur->subscriber != write_to) { + cur = cur->next; + continue; + } + return !cur->paused; + } + return false; +} + +/* + * Returns true if the subscription is active, but paused. + */ +bool +sb_route_is_paused(switchboard_t *ctx, party_t *read_from, party_t *write_to) +{ + if (read_from == NULL || write_to == NULL) { + return false; + } + + if (!read_from->open_for_read || !write_to->open_for_write) { + return false; + } + + fd_party_t *reader = get_fd_obj(read_from); + subscription_t *cur = reader->subscribers; + + while (cur != NULL) { + if (cur->subscriber != write_to) { + cur = cur->next; + continue; + } + return cur->paused; + } + return false; +} + +/* + * Returns true if the subscription is active, whether or not it is + * paused. + */ +bool +sb_is_subscribed(switchboard_t *ctx, party_t *read_from, party_t *write_to) +{ + if (read_from == NULL || write_to == NULL) { + return false; + } + + if (!read_from->open_for_read || !write_to->open_for_write) { + return false; + } + + fd_party_t *reader = get_fd_obj(read_from); + subscription_t *cur = reader->subscribers; + + while (cur != NULL) { + if (cur->subscriber != write_to) { + cur = cur->next; + continue; + } + return true; + } + return false; +} + + /* * Initializes a switchboard object, primarily zeroing out the * contents, and setting up message buffering. @@ -844,19 +1017,22 @@ handle_one_read(switchboard_t *ctx, party_t *party) while (sublist != NULL) { party_t *sub = sublist->subscriber; - switch(sub->party_type) { - case PT_FD: - publish(ctx, buf, read_result, sub); - break; - case PT_STRING: - add_data_to_string_out(get_dstr_obj(sub), buf, read_result); - break; - case PT_CALLBACK: - (*sub->info.cbinfo.callback)(ctx->extra, sub->extra, buf, - (size_t)read_result); - break; - default: - break; + + if (!sublist->paused) { + switch(sub->party_type) { + case PT_FD: + publish(ctx, buf, read_result, sub); + break; + case PT_STRING: + add_data_to_string_out(get_dstr_obj(sub), buf, read_result); + break; + case PT_CALLBACK: + (*sub->info.cbinfo.callback)(ctx->extra, sub->extra, buf, + (size_t)read_result); + break; + default: + break; + } } sublist = sublist->next; } @@ -885,7 +1061,7 @@ handle_one_accept(switchboard_t *ctx, party_t *party) if (errno == EINTR || errno == EAGAIN) { continue; } - if (errno == ECONNABORTED || errno == EWOULDBLOCK) { + if (errno == ECONNABORTED) { break; } party->found_errno = errno; @@ -1192,11 +1368,9 @@ is_registered_writer(switchboard_t *ctx, party_t *target) } /* - * Dealloc any memory we're responsible for. Gets called - * automatically at the end of sb_operate_switchboard(), but - * must be invoked manually if you don't use that. + * Dealloc any memory we're responsible for. * - * Also note that this does NOT free the switchboard object, + * Note that this does NOT free the switchboard object, * just any internal data structures. */ void @@ -1274,18 +1448,22 @@ sb_destroy(switchboard_t *ctx, bool free_parties) } /* - * Extract results from the switchbaord; does not do any cleanup itself; - * you will still need to free the switchboard if it's heap alloc'd. + * Extract results from the switchbaord. */ void -sb_prepare_results(switchboard_t *ctx) +sb_get_results(switchboard_t *ctx, sb_result_t *result) { - sb_result_t *cur; - str_dst_party_t *strobj; - party_t *party = ctx->party_loners; // Look for string outputs. - int capcount = 0; - int ix = 0; + str_dst_party_t *strobj; + party_t *party = ctx->party_loners; // Look for str outputs. + int capcount = 0; + int ix = 0; + if (result->inited) { + return; + } + + result->inited = true; + while (party) { if (party->party_type == PT_STRING && party->can_write_to_it) { capcount++; @@ -1293,24 +1471,24 @@ sb_prepare_results(switchboard_t *ctx) party = party->next_loner; } - ctx->result.num_captures = capcount; - ctx->result.captures = calloc(sizeof(capture_result_t), capcount+1); + result->num_captures = capcount; + result->captures = calloc(sizeof(capture_result_t), capcount+1); party = ctx->party_loners; while (party) { if (party->party_type == PT_STRING && party->can_write_to_it) { - capture_result_t *r = ctx->result.captures + ix; + capture_result_t *r = result->captures + ix; strobj = get_dstr_obj(party); r->tag = strobj->tag; r->len = strobj->ix; - + if (strobj->ix) { - char *s = (char *)calloc(strobj->len, 1); - memcpy(s, strobj->strbuf, strobj->ix); - r->contents = s; + r->contents = strobj->strbuf; + strobj->strbuf = 0; + strobj->ix = 0; } else { r->contents = NULL; } @@ -1320,6 +1498,40 @@ sb_prepare_results(switchboard_t *ctx) } } +char * +sb_result_get_capture(sb_result_t *ctx, char *tag, bool caller_borrow) +{ + char *result; + + for (int i = 0; i < ctx->num_captures; i++) { + if (!strcmp(ctx->captures[i].tag, tag)) { + result = ctx->captures[i].contents; + + if (!caller_borrow) { + ctx->captures[i].contents = NULL; + } + return result; + } + } + return NULL; +} + +/* + * The tags are borrowed, so we don't free. If you call this, then + * you're asking to free the capture string copies and the array of + * captures, but the actual sb_result_t object wasn't allocated by + * this API, so we don't own it and this does not try to free it. + */ +void +sb_result_destroy(sb_result_t *ctx) { + for (int i = 0; i < ctx->num_captures; i++) { + if(ctx->captures[i].contents) { + free(ctx->captures[i].contents); + } + } + free(ctx->captures); +} + /* * Returns true if there are any open writers that have enqueued items. */ @@ -1369,21 +1581,3 @@ sb_operate_switchboard(switchboard_t *ctx, bool loop) } while(loop); return false; } - -/* - * Operates a setup switchboard, returning a result and dealing w/ - * memory management on exit. - */ -sb_result_t * -sb_automatic_switchboard(switchboard_t *ctx, bool free_party_objects) -{ - - sb_result_t *result = (sb_result_t *)malloc(sizeof(sb_result_t)); - - sb_operate_switchboard(ctx, true); - sb_prepare_results(ctx); - memcpy(result, &ctx->result, sizeof(sb_result_t)); - sb_destroy(ctx, free_party_objects); - - return result; -} diff --git a/nimutils/switchboard.h b/nimutils/c/switchboard.h similarity index 92% rename from nimutils/switchboard.h rename to nimutils/c/switchboard.h index 4848956..ae8d2e8 100644 --- a/nimutils/switchboard.h +++ b/nimutils/c/switchboard.h @@ -74,6 +74,7 @@ typedef struct sb_heap_t { typedef struct subscription_t { struct subscription_t *next; struct party_t *subscriber; + bool paused; } subscription_t; /* @@ -217,6 +218,7 @@ typedef struct { } capture_result_t; typedef struct { + bool inited; int num_captures; capture_result_t *captures; } sb_result_t; @@ -244,7 +246,6 @@ typedef struct switchboard_t { size_t heap_elems; void *extra; bool ignore_running_procs_on_shutdown; - sb_result_t result; } switchboard_t; @@ -277,6 +278,7 @@ typedef struct { party_t capture_stdout; party_t capture_stderr; void (*startup_callback)(void *); + sb_result_t result; struct termios saved_termcap; struct termios *parent_termcap; struct termios *child_termcap; @@ -307,13 +309,13 @@ extern void sb_init_party_listener(switchboard_t *, party_t *, int, accept_cb_t, bool, bool); extern party_t * sb_new_party_listener(switchboard_t *, int, accept_cb_t, bool, bool); -extern void sb_init_party_fd(switchboard_t *, party_t *, int , int , bool, - bool); +extern void sb_init_party_fd(switchboard_t *, party_t *, int , int , bool, bool); extern party_t *sb_new_party_fd(switchboard_t *, int, int, bool, bool); -extern void sb_init_party_input_buf(switchboard_t *, party_t *, char *, size_t, - bool, bool); -extern party_t *sb_new_party_input_buf(switchboard_t *, char *, size_t, bool, - bool); +extern void sb_init_party_input_buf(switchboard_t *, party_t *, char *, + size_t, bool, bool, bool); +extern party_t *sb_new_party_input_buf(switchboard_t *, char *, size_t, + bool, bool, bool); +extern void sb_party_input_buf_new_string(party_t *, char *, size_t, bool, bool); extern void sb_init_party_output_buf(switchboard_t *, party_t *, char *, size_t); extern party_t *sb_new_party_output_buf(switchboard_t *, char *, size_t); @@ -327,13 +329,19 @@ extern void sb_set_extra(switchboard_t *, void *); extern void *sb_get_party_extra(party_t *); extern void sb_set_party_extra(party_t *, void *); extern bool sb_route(switchboard_t *, party_t *, party_t *); +extern bool sb_pause_route(switchboard_t *, party_t *, party_t *); +extern bool sb_resume_route(switchboard_t *, party_t *, party_t *); +extern bool sb_route_is_active(switchboard_t *, party_t *, party_t *); +extern bool sb_route_is_paused(switchboard_t *, party_t *, party_t *); +extern bool sb_route_is_subscribed(switchboard_t *, party_t *, party_t *); extern void sb_init(switchboard_t *, size_t); extern void sb_set_io_timeout(switchboard_t *, struct timeval *); extern void sb_clear_io_timeout(switchboard_t *); extern void sb_destroy(switchboard_t *, bool); -extern void sb_prepare_results(switchboard_t *); extern bool sb_operate_switchboard(switchboard_t *, bool); -extern sb_result_t *sb_automatic_switchboard(switchboard_t *, bool); +extern void sb_get_results(switchboard_t *, sb_result_t *); +extern char *sb_result_get_capture(sb_result_t *, char *, bool); +extern void sb_result_destroy(sb_result_t *); extern void subproc_init(subprocess_t *, char *, char *[]); extern bool subproc_set_envp(subprocess_t *, char *[]); extern bool subproc_pass_to_stdin(subprocess_t *, char *, size_t, bool); @@ -348,7 +356,6 @@ extern bool subproc_set_startup_callback(subprocess_t *, void (*)(void *)); extern int subproc_get_pty_fd(subprocess_t *); extern void subproc_start(subprocess_t *); extern bool subproc_poll(subprocess_t *); -extern void subproc_prepare_results(subprocess_t *); extern void subproc_run(subprocess_t *); extern void subproc_close(subprocess_t *); extern pid_t subproc_get_pid(subprocess_t *); @@ -362,6 +369,10 @@ extern void subproc_set_child_termcap(subprocess_t *, struct termios *); extern void subproc_set_extra(subprocess_t *, void *); extern void *subproc_get_extra(subprocess_t *); extern int subproc_get_pty_fd(subprocess_t *); +extern void pause_passthrough(subprocess_t *, unsigned char); +extern void resume_passthrough(subprocess_t *, unsigned char); +extern void pause_capture(subprocess_t *, unsigned char); +extern void resume_capture(subprocess_t *, unsigned char); extern void termcap_get(struct termios *); extern void termcap_set(struct termios *); extern void termcap_set_raw_mode(struct termios *); diff --git a/nimutils/test.c b/nimutils/c/test.c similarity index 100% rename from nimutils/test.c rename to nimutils/c/test.c diff --git a/nimutils/colortable.nim b/nimutils/colortable.nim index 9218409..1494a27 100644 --- a/nimutils/colortable.nim +++ b/nimutils/colortable.nim @@ -2,6 +2,12 @@ # https://en.wikipedia.org/wiki/Web_colors import tables, os, parseUtils +## This array contains all names we recognize for full 24-bit color. +## APIs that accept color names will also accept #abc123 style hex +## codes, but the algorithm for converting down to 256 colors may +## not be awesome, depending on your terminal, so color names are +## preferable when possible (all colors here should have a 'close' +## equal in the 256-color (8-bit) version below. var colorTable* = { "mediumvioletred" : 0xc71585, "deeppink" : 0xff1493, @@ -161,6 +167,7 @@ var colorTable* = { # Someone should script up some A/B testing to hone in on some of # these better, but this all looks good enough for now. +## 8-bit color mappings for our named colors. var color8Bit* = { "mediumvioletred" : 126, "deeppink" : 206, # Pretty off still. 201, 199 @@ -317,24 +324,53 @@ template getColorTable*(): OrderedTable = colorTable template get8BitTable*(): OrderedTable = color8Bit proc setShowColor*(val: bool) = + ## If this is set to `false`, then rope "rendering" will not apply + ## any ansi codes whatsoever. + ## + ## All output should otherwise look the same as it would with color, + ## meaning there will be no variations in width. showColor = val proc getShowColor*(): bool = + ## Returns whether color will be output. return showColor proc setUnicodeOverAnsi*(val: bool) = + ## In some cases, we can use Unicode to attempt to draw some features, + ## like underlining. This is generally good because people might turn + ## colors off, in which case we interpret that as *no ansi codes*. + ## Plus, some terminals may not support these things. + ## + ## Unfortunately, not all fonts render unicode underlining well either. + ## Thus, the toggle. unicodeOverAnsi = val proc getUnicodeOverAnsi*(): bool = + ## Returns whether we're preferring unicode over ansi, when possible. return unicodeOverAnsi proc setColor24Bit*(val: bool) = + ## Sets whether to use 24-bit color. If the terminal doesn't support + ## it, you won't like the results. If you turn this off, you'll only + ## get 256 colors, but you can be sure the terminal will render + ## okay. + ## + ## We do have limited auto-detection based on environment variables, + ## but it isn't very good at this point. color24Bit = val proc getColor24Bit*(): bool = + ## Returns true for 24-bit output, false for 8-bit. return color24Bit proc autoDetectTermPrefs*() = + ## Does very limited detection of terminal capabilities. We'd + ## eventually like to do deeper terminal queries. For now, you're + ## mainly on your own, sorry :/ + ## + ## This is run once automatically when importing colortable, But you + ## can call it again if you like! I don't know why you would, since + ## you're in control of any environment variable changes, though :) # TODO: if on a TTY, query the terminal for trycolor support, # per the `Checking for colorterm` section of: # https://github.com/termstandard/colors @@ -354,7 +390,12 @@ proc autoDetectTermPrefs*() = unicodeOverAnsi = false proc hexColorTo8Bit*(hex: string): int = - # Returns -1 if invalid. + ## Implements the algorithmic mapping of the 24-bit color space to 8 + ## bit color. Returns -1 if the input was invalid. + ## + ## Since terminals will generally use a pallate for 256 colors, the + ## mappings usually aren't exact, and can occasionally be WAY off + ## what is expected. var color: int if parseHex(hex, color) != 6: @@ -370,6 +411,8 @@ proc hexColorTo8Bit*(hex: string): int = int(blue * 3 / 255) proc colorNameToHex*(name: string): (int, int, int) = + ## Looks up a color name, and converts it to the three raw (r, g, b) + ## values that we need for ANSI codes. let colorTable = getColorTable() var color: int @@ -380,6 +423,8 @@ proc colorNameToHex*(name: string): (int, int, int) = result = (color shr 16, (color shr 8) and 0xff, color and 0xff) proc colorNameToVga*(name: string): int = + ## Looks up a color name, and converts it to the 8-bit color value + ## mapped to that name. let color8Bit = get8BitTable() if name in color8Bit: @@ -387,4 +432,5 @@ proc colorNameToVga*(name: string): int = else: return hexColorTo8Bit(name) -autoDetectTermPrefs() +once: + autoDetectTermPrefs() diff --git a/nimutils/crownhash.nim b/nimutils/crownhash.nim new file mode 100644 index 0000000..8e109b0 --- /dev/null +++ b/nimutils/crownhash.nim @@ -0,0 +1,588 @@ +## This wraps my C-based lock-free, wait-free hash table +## implementation. I don't know much about Nim's multi-threading at +## this point, made worse by the fact that there are so many different +## memory managers in Nim. +## +## So I'm not sure how to use it (you'll definitely want anything +## passed in to be shared memory), nor do I know what memory managers +## it will work with. +## +## That's because my purpose for wrapping this is to enable the con4m +## data store to be accessible via C API, from mulitple +## threads. Con4m, for the forseeable future, should only ever need a +## single thread accessing the store. +## +## This hash table does require any new thread to call registerThread(). +## +## Memory management approach: +## +## For most primitive types, we store a 64-bit value that neither Nim +## nor the underlying implementation needs to memory manage. That includes +## all int and float types, bools, enums, etc. +## +## For objects (things that are references and should thus be heap +## allocated in nim) we can GC_ref() them and store them directly. +## +## Strings in nim are generally three-tier. There's a stack-allocated +## value that has the length, and a pointer to a heap-allocated value, +## which contains a capacity and a pointer to the actual c-string. +## +## To deal with this, we create our own heap-allocated data structure +## that keeps the data we need to re-assemble the string when we remove +## it. We also GC_ref() the heap-allocated payload (and unref it when +## ejecting). +## +## We could also ensure a heap-allocated string by sticking it inside +## of a ref object and copying it in, but it's an extra layer of +## indirection for computing the hash value... for strings, we want to +## do that by treating it as a null-terminated C string, not a pointer. +## +## With C strings, we currently bundle them in a Nim string to simplify +## memory management. This may change soon, so don't count on it. Note +## here that, in Nim's FFI, $(x) copies the string, but cstring(s) does +## not. The 'sink' modifier passes 'ownership'. +## +## For everything else, we'll generally see it stack-allocated, and +## won't necessarily have access to a consistent storage location, even +## if it never leaves the stack. That makes such data objects +## unsuitable for being hash keys (though, we could support custom +## per-type hash functions in the future). +## +## However, we can store such things as *values* by wrapping them in a +## heap allocated object that we then incref. We'll then hold a copy of +## that object. +## +## When anything gets ejected from the hash table other than a +## primitive ordinal or float type, we GC_unref() if it was allocated +## from Nim, and currently ignore otherwise. + + +import sugar, os, macros, options + +{.pragma: hatc, cdecl, importc.} + +type + Dict*[T, V] {. importc: "hatrack_dict_t", header: "crownhash.h", nodecl.} = object + DictRef*[T, V] = ref Dict[T, V] + DictKeyType = enum + KTInt, KTFloat, KtCStr, KtPtr, KtObjInt, KtObjReal, KtObjCstr, + KtObjPtr, KtObjCustom, KtForce32Bits = 0x0fffffff + StackBox[T] = ref object + contents: T + ownedByNim: bool + StrBoxObj = object + data: cstring + str: string + ownedByNim: bool + StrBox = ref StrBoxObj + StrPayloadCast = object + cap: int + data: UncheckedArray[char] + StrCast = object + len: int + p: ptr StrPayloadCast + SomeString = string | cstring # String box. + SomeRef = ref or pointer # Not boxed. + SomeNumber = SomeOrdinal or SomeFloat + RawItem = object + key: pointer + value: pointer + + +proc toStrBox(s: string): StrBox = + var + outer = cast[StrCast](s) + inner = cast[ref StrPayloadCast](outer.p) + + new result + + result.data = cstring(s) + result.str = s + result.ownedByNim = true + GC_ref(result) + +proc toStrBox(s: cstring): StrBox = + return toStrBox($(s)) + +proc unboxStr(s: StrBox): string = + return s.str + +proc ejectStrBox(s: StrBox) = + if s.ownedByNim: + GC_unref(s) + +proc toStackBox[T](o: T): StackBox[T] = + result = StackBox[T](contents: o, ownedByNim: true) + GC_ref(result) + +proc unboxStackObj[T](box: StackBox[T]): T = + return box.contents + +proc ejectStackBox[T](s: StackBox[T]) = + if s.ownedByNim: + GC_unref(s) + +proc hatrack_dict_init(ctx: var Dict, key_type: DictKeyType) {.hatc.} +proc hatrack_dict_cleanup(ctx: ptr Dict) {.hatc.} +proc hatrack_dict_set_consistent_views(ctx: var Dict, yes: bool) {.hatc.} +proc hatrack_dict_get_consistent_views(ctx: var Dict): bool {.hatc.} +proc hatrack_dict_set_hash_offset(ctx: var Dict, offset: cint) {.hatc.} +proc hatrack_dict_get(ctx: var Dict, key: pointer, found: var bool): + pointer {.hatc.} +proc hatrack_dict_put(ctx: var Dict, key: pointer, + value: pointer) {. + hatc.} +proc hatrack_dict_replace(ctx: var Dict, key: pointer, value: pointer): + bool {.hatc.} +proc hatrack_dict_add(ctx: var Dict, key: pointer, value: pointer): + bool {.hatc.} +proc hatrack_dict_remove(ctx: var Dict, key: pointer): bool {.hatc.} +proc hatrack_dict_keys_sort(ctx: var Dict, n: ptr uint64): + pointer {.hatc.} +proc hatrack_dict_values_sort(ctx: var Dict, n: ptr uint64): + pointer {.hatc.} +proc hatrack_dict_items_sort(ctx: var Dict, n: ptr uint64): + pointer {.hatc.} +proc hatrack_dict_keys_nosort(ctx: var Dict, n: ptr uint64): + pointer {.hatc.} +proc hatrack_dict_values_nosort(ctx: var Dict, n: ptr uint64): + pointer {.hatc.} +proc hatrack_dict_items_nosort(ctx: var Dict, n: ptr uint64): + pointer {.hatc.} +proc hatrack_dict_set_free_handler[T, V](ctx: var Dict[T, V], + cb: (var Dict[T, V], ptr RawItem) -> void) {.hatc.} +proc register_thread() {.cdecl, importc: "mmm_register_thread" .} + +proc decrefDictItems[T, V](dict: var Dict[T, V], p: ptr RawItem) = + when T is SomeString: + ejectStrBox(cast[StrBox](p[].key)) + elif T is ref: + GC_unref(cast[T](p[].value)) + + when V is SomeString: + ejectStrBox(cast[StrBox](p[].value)) + elif V is ref: + GC_unref(cast[V](p[].value)) + elif not (V is SomeNumber or V is pointer): + ejectStackBox(cast[StackBox[V]](p[].value)) +once: + # Auto-register the main thread. + registerThread() + +proc initDict*[T, V](dict: var Dict[T, V]) = + ## Initialize a Dict. + when T is SomeOrdinal: + hatrack_dict_init(dict, KtInt) + elif T is SomeFloat: + hatrack_dict_init(dict, KtFloat) + elif T is SomeString: + hatrack_dict_init(dict, KtObjCStr) + hatrack_dict_set_hash_offset(dict, 0) + elif T is SomeRef: + hatrack_dict_init(dict, KtPtr) + else: + static: + error("Cannot currently have keys of seq or object types") + + dict.hatrack_dict_set_consistent_views(true) + when not (T is SomeNumber and V is SomeNumber): + dict.hatrack_dict_set_free_handler(decrefDictItems) + +proc `=destroy`*[T, V](x: Dict[T, V]) = + ## Calls the underlying C cleanup routine to deallocate everything + ## specifically allocated in C. + hatrack_dict_cleanup(addr x) + + +proc `[]=`*[T, V](dict: var Dict[T, V], key: T, value: sink V) = + ## This assigns, whether or not there was a previous value + ## associated with the passed key. + if not dict.hatrack_dict_get_consistent_views(): + initDict[T, V](dict) + var p: pointer + when T is SomeString: + p = cast[pointer](key.toStrBox()) + else: + p = cast[pointer](key) + + when V is SomeOrdinal: + dict.hatrack_dict_put(p, cast[pointer](int64(value))) + elif V is SomeFloat: + dict.hatrack_dict_put(p, cast[pointer](float(value))) + elif V is SomeString: + dict.hatrack_dict_put(p, cast[pointer](value.toStrBox())) + elif V is ref: + GC_ref(value) + dict.hatrack_dict_put(p, cast[pointer](value)) + elif V is pointer: + dict.hatrack_dict_put(p, cast[pointer](value)) + else: + dict.hatrack_dict_put(p, cast[pointer](value.toStackBox())) + +proc `[]=`*[T, V](dict: DictRef[T, V], key: T, value: sink V) = + `[]=`(dict[], key, value) + +proc replace*[T, V](dict: var Dict[T, V], key: T, value: sink V): bool = + ## This replaces the value associated with a given key. If the key + ## has not yet been set, then `false` is returned and no value is + ## set. + if not dict.hatrack_dict_get_consistent_views(): + initDict[T, V](dict) + + var p: pointer + when T is SomeString: + p = cast[pointer](key.toStrBox()) + else: + p = cast[pointer](key) + + when V is SomeOrdinal: + return dict.hatrack_dict_replace(p, cast[pointer](int64(value))) + elif V is SomeFloat: + return dict.hatrack_dict_replace(p, cast[pointer](float(value))) + elif V is SomeString: + return dict.hatrack_dict_replace(p, cast[pointer](value.toStrBox())) + elif V is ref: + GC_ref(value) + return dict.hatrack_dict_replace(p, cast[pointer](value)) + elif V is pointer: + return dict.hatrack_dict_replace(p, cast[pointer](value)) + else: + return dict.hatrack_dict_replace(p, cast[pointer](value.toStackBox())) + +proc replace*[T, V](dict: DictRef[T, V], key: T, value: sink V): bool = + return replace(dict[], key, value) + +proc add*[T, V](dict: var Dict[T, V], key: T, value: sink V): bool = + ## This sets a value associated with a given key, but only if the + ## key does not exist in the hash table at the time of the + ## operation. + + if not dict.hatrack_dict_get_consistent_views(): + initDict[T, V](dict) + + var p: pointer + when T is SomeString: + p = cast[pointer](key.toStrBox()) + else: + p = cast[pointer](key) + + when V is SomeOrdinal: + return dict.hatrack_dict_replace(p, cast[pointer](int64(value))) + elif V is SomeFloat: + return dict.hatrack_dict_replace(p, cast[pointer](float(value))) + elif V is SomeString: + return dict.hatrack_dict_replace(p, cast[pointer](value.toStrBox())) + elif V is ref: + GC_ref(value) + return dict.hatrack_dict_replace(p, cast[pointer](value)) + elif V is pointer: + return dict.hatrack_dict_replace(p, cast[pointer](value)) + else: + return dict.hatrack_dict_replace(p, cast[pointer](value.toStackBox())) + +proc add*[T, V](dict: DictRef[T, V], key: T, value: sink V): bool = + return add(dict[], key, value) + +proc lookup*[T, V](dict: var Dict[T, V], key: T): Option[V] = + ## Retrieve the value associated with a key, wrapping it in + ## an option. If the key isn't present, then returns `none`. + ## + ## See the [] operator for a version that throws an exception + ## if the key is not present in the table. + + if not dict.hatrack_dict_get_consistent_views(): + initDict[T, V](dict) + + var + found: bool + p: pointer + + when T is SomeString: + p = cast[pointer](key.toStrBox()) + else: + p = cast[pointer](key) + + var retp = dict.hatrack_dict_get(p, found) + + if found: + when V is SomeOrdinal: + var x: int64 = cast[int64](retp) + result = some(V(x)) + elif V is SomeFloat: + var x: float = cast[float](retp) + result = some(V(x)) + elif V is string: + var box = cast[StrBox](retp) + result = some(box.unboxStr()) + elif V is cstring: + var + box = cast[StrBox](retp) + str = box.unboxStr() + result = some(cstring(str)) + elif V is SomeRef: + # No need to worry about possible incref; the type will cause + # the right thing to happen here. + result = some(cast[V](retp)) + else: + var box = cast[StackBox[V]](retp) + result = some(box.contents) + +proc contains*[T, V](dict: var Dict[T, V], key: T): bool = + ## In a multi-threaded environment, this shouldn't be used when + ## there might be any sort of race condition. Use lookup() instead. + return dict.lookup(key).isSome() + +proc contains*[T, V](dict: DictRef[T, V], key: T): bool = + return contains(dict[], key) + +proc `[]`*[T, V](dict: var Dict[T, V], key: T) : V = + ## Retrieve the value associated with a key, or else throws an erro + ## if it's not present. + ## + ## See `lookup` for a version that returns an Option, and thus + ## will not throw an error when the item is not found. + + var optRet: Option[V] = dict.lookup(key) + + if optRet.isSome(): + return optRet.get() + else: + raise newException(KeyError, "Dictionary key was not found.") + +proc `[]`*[T, V](dict: DictRef[T, V], key: T) : V = + return `[]`(dict[], key) + +proc toDict*[T, V](pairs: openarray[(T, V)]): DictRef[T, V] = + ## Use this to convert a nim {} literal to a Dict. + result = DictRef[T, V]() + initDict(result[]) + for (k, v) in pairs: + result[k] = v + +proc newDict*[T, V](): DictRef[T, V] = + ## Heap-allocate a DictRef + result = DictRef[T, V]() + initDict[T, V](result[]) + +proc del*[T, V](dict: var Dict[T, V], key: T): bool {.discardable.} = + ## Deletes any value associated with a given key. + ## + ## Note that this does *not* throw an exception if the item is not present, + ## as multiple threads might be attempting parallel deletes. Instead, + ## if you care about the op succeeding, check the return value. + if not dict.hatrack_dict_get_consistent_views(): + initDict[T, V](dict) + + var + p: pointer + + when T is SomeString: + p = cast[pointer](key.toStrBox()) + else: + p = cast[pointer](key) + + return dict.hatrack_dict_remove(p) + +proc delete*[T, V](dict: var Dict[T, V], key: T): bool {.discardable.} = + return del[T, V](dict, key) + +proc del*[T, V](dict: DictRef[T, V], key: T): bool {.discardable.} = + return del[T, V](dict[], key) + +proc delete*[T, V](dict: DictRef[T, V], key: T): bool {.discardable.} = + return del[T, V](dict[], key) + +proc keys*[T, V](dict: var Dict[T, V], sort = false): seq[T] = + ## Returns a consistent view of all keys in a dictionary at some + ## moment in time during the execution of the function. + ## + ## Note that this is *not* an iterator. This is intentional. The + ## only way to get a consistent view in a parallel environment is to + ## create a consistent copy; we already have the copy, so having an + ## extra layer of cursor state is definitely not needed. + ## + ## Memory is cheap and plentyful; you'll survive. + + if not dict.hatrack_dict_get_consistent_views(): + initDict[T, V](dict) + + when T is SomeString: + var p: ptr UncheckedArray[StrBox] + elif T is SomeOrdinal: + var p: ptr UncheckedArray[int64] + elif T is SomeFloat: + var p: ptr UncheckedArray[float] + else: + var p: ptr UncheckedArray[T] + + var + n: uint64 + + if sort: + p = cast[typeof(p)](hatrack_dict_keys_sort(dict, addr n)) + else: + p = cast[typeof(p)](hatrack_dict_keys_nosort(dict, addr n)) + + for i in 0 ..< n: + when T is string: + result.add(unboxStr(p[i])) + elif T is cstring: + result.add(cstring(unboxStr(p[i]))) + else: + result.add(T(p[i])) + +proc keys*[T, V](dict: DictRef[T, V], sort = false): seq[T] = + return keys[T, V](dict[], sort) + +proc values*[T, V](dict: var Dict[T, V], sort = false): seq[V] = + ## Returns a consistent view of all values in a dictionary at some + ## moment in time during the execution of the function. + ## + ## Note that this is *not* an iterator. This is intentional. The + ## only way to get a consistent view in a parallel environment is to + ## create a consistent copy; we already have the copy, so having an + ## extra layer of cursor state is definitely not needed. + ## + ## Memory is cheap and plentyful; you'll survive. + + if not dict.hatrack_dict_get_consistent_views(): + initDict[T, V](dict) + + when V is SomeOrdinal: + var + p: ptr UncheckedArray[int64] + elif V is SomeFloat: + var + p: ptr UncheckedArray[float] + elif V is SomeRef: + var + p: ptr UncheckedArray[V] + elif V is SomeString: + var + p: ptr UncheckedArray[StrBox] + else: + var + p: ptr UncheckedArray[StackBox[V]] + + var n: uint64 + + if sort: + p = cast[typeof(p)](hatrack_dict_values_sort(dict, addr n)) + else: + p = cast[typeof(p)](hatrack_dict_values_nosort(dict, addr n)) + + for i in 0 ..< n: + when V is SomeOrdinal or V is SomeFloat or V is SomeRef: + result.add(V(p[i])) + elif V is string: + result.add(unboxStr(p[i])) + elif V is cstring: + result.add(cstring(unboxStr(p[i]))) + else: + result.add(unboxStackObj[V](p[i])) + +proc values*[T, V](dict: DictRef[T, V], sort = false): seq[V] = + return values[T, V](dict[], sort) + +proc items*[T, V](dict: var Dict[T, V], sort = false): seq[(T, V)] = + ## Returns a consistent view of all key, value pairs in a dictionary + ## at some moment in time during the execution of the function. + ## + ## Note that this is *not* an iterator. This is intentional. The + ## only way to get a consistent view in a parallel environment is to + ## create a consistent copy; we already have the copy, so having an + ## extra layer of cursor state is definitely not needed. + ## + ## Memory is cheap and plentyful; you'll survive. + + if not dict.hatrack_dict_get_consistent_views(): + initDict[T, V](dict) + + var + p: ptr UncheckedArray[RawItem] + n: uint64 + item: tuple[key: T, value: V] + + if sort: + p = cast[typeof(p)](hatrack_dict_items_sort(dict, addr n)) + else: + p = cast[typeof(p)](hatrack_dict_items_nosort(dict, addr n)) + + for i in 0 ..< n: + var uncast = p[i] + + when T is string: + item.key = unboxStr(cast[StrBox](uncast.key)) + elif T is cstring: + item.key = cstring(unboxStr(cast[StrBox](uncast.key))) + elif T is SomeOrdinal: + item.key = T(cast[int64](uncast.key)) + elif T is SomeFloat: + item.key = T(cast[float](uncast.key)) + else: # T is SomeRef + item.key = T(uncast.key) + + when V is string: + item.value = unboxStr(cast[StrBox](uncast.value)) + elif V is cstring: + item.value = cstring(unboxStr(cast[StrBox](uncast.value))) + elif V is SomeOrdinal: + item.value = V(cast[int64](uncast.value)) + elif V is SomeFloat: + item.value = T(cast[float](uncast.value)) + elif V is SomeRef: + item.value = cast[V](uncast.value) + else: + item.value = unboxStackObj[V](cast[StackBox[V]](uncast.value)) + + result.add(item) + +proc items*[T, V](dict: DictRef[T, V], sort = false): seq[(T, V)] = + return items[T, V](dict[], sort) + +proc `$`*[T, V](dict: var Dict[T, V]): string = + let view = dict.items() + result = "{ " + for i, (k, v) in view: + if i != 0: + result &= ", " + when T is SomeString: + result &= "\"" & $(k) & "\" : " + else: + result &= $(k) & " : " + when V is SomeString: + result &= "\"" & $(v) & "\"" + else: + result &= $(v) + result &= " }" + +proc `$`*[T, V](dict: DictRef[T, V]): string = + return `$`[T, V](dict[]) + +proc deepEquals*[T, V](dict1: var Dict[T, V], dict2: var Dict[T, V]): bool = + ## This operation doesn't make too much sense in most cases; we'll + ## leave == to default to a pointer comparison (for dictrefs). + let + view1 = dict1.items[T, V](sort = true) + view2 = dict2.items[T, V](sort = true) + + if view1.len() != view2.len(): + return false + + for i in 0 ..< view1.len(): + let + (k1, v1) = view1[i] + (k2, v2) = view2[i] + if k1 != k2 or v1 != v2: + return false + + return true + +proc deepEquals*[T, V](dict1: var Dict[T, V], dict2: DictRef[T, V]): bool = + return deepEquals[T, V](dict1, dict2[]) + +proc deepEquals*[T, V](dict1: DictRef[T, V], dict2: var Dict[T, V]): bool = + return deepEquals[T, V](dict1[], dict2) + +proc deepEquals*[T, V](dict1: DictRef[T, V], dict2: DictRef[T, V]): bool = + return deepEquals[T, V](dict1[], dict2[]) diff --git a/nimutils/dict.nim b/nimutils/dict.nim new file mode 100644 index 0000000..4241d8c --- /dev/null +++ b/nimutils/dict.nim @@ -0,0 +1,6 @@ +## When we also hook up our single-threaded implementation, this will +## multiplex the Dict interface statically, depending on whether or +## not threads are used. +when compileOption("threads"): + import crownhash + export crownhash diff --git a/nimutils/either.nim b/nimutils/either.nim index 106d01d..b7921b1 100644 --- a/nimutils/either.nim +++ b/nimutils/either.nim @@ -1,6 +1,11 @@ ## :Author: John Viega (john@crashoverride.com) ## :Copyright: 2023, Crash Override, Inc. ## +## This was an experiment that didn't really work well; it needs a lot +## more work to be useful, so please don't use this; the only reason I +## haven't removed it is because there is code in Con4m that is still +## using it for the time being. + import macros type diff --git a/nimutils/encodings.nim b/nimutils/encodings.nim index 17cd4d3..9ca7b6d 100644 --- a/nimutils/encodings.nim +++ b/nimutils/encodings.nim @@ -106,7 +106,7 @@ macro declB32Decoder(modifier: static[string], mapname: untyped): untyped = d32 = ident("d32" & modifier) return quote do: - proc `d32`(c: char): uint {.inline.} = + proc `d32`*(c: char): uint {.inline.} = if int(c) > 90 or int(c) < 48: raise newException(ValueError, "Invalid b32 char") let ix = if int(c) >= 65: int(c) - 55 else: int(c) - 48 @@ -198,6 +198,9 @@ template oneChrTS() = mask64 = mask64 shr 5 proc encodeUlid*(ts: uint64, randbytes: openarray[char], dash = true): string = + ## encode a ULID, passing in the specific non-timestamp + ## bytes. Generally, you should instead use `getUlid()`, which uses + ## random bytes, as is typical. var str = "" mask64 = 0x3e00000000000'u64 @@ -210,6 +213,7 @@ proc encodeUlid*(ts: uint64, randbytes: openarray[char], dash = true): string = result = str & base32vEncode(randbytes[0 ..< 10]) proc getUlid*(dash = true): string = + ## Returns a unique ULID, per the standard. var randbytes = secureRand[array[10, char]]() ts = unixTimeInMs() @@ -217,7 +221,11 @@ proc getUlid*(dash = true): string = encodeUlid(ts, randbytes) proc ulidToTimeStamp*(s: string): uint64 = - ## No error checking done on purpose. + ## Extracts a timestamp from a ULID, measured in miliseconds since + ## the start of 1970 UTC. + ## + ## We do no error checking; if you don't pass in an actual ULID, you + ## won't like your results. result = uint64(d32v(s[0])) shl 45 result = result or uint64(d32v(s[1])) shl 40 result = result or uint64(d32v(s[2])) shl 35 @@ -228,47 +236,3 @@ proc ulidToTimeStamp*(s: string): uint64 = result = result or uint64(d32v(s[7])) shl 10 result = result or uint64(d32v(s[8])) shl 5 result = result or uint64(d32v(s[9])) - - -when isMainModule: - let x = getUlid() - echo unixTimeInMs() - echo x, " ", x.ulidToTimeStamp() - let y = getUlid() - echo y, " ", y.ulidToTimeStamp() - echo unixTimeInMs() - echo base32Encode("This is some string.") - echo "KRUGS4ZANFZSA43PNVSSA43UOJUW4ZZO (is the answer)" - echo base32Encode("This is some string") - echo "KRUGS4ZANFZSA43PNVSSA43UOJUW4ZY (is the answer)" - echo base32Encode("This is some strin") - echo "KRUGS4ZANFZSA43PNVSSA43UOJUW4 (is the answer)" - echo base32Encode("This is some stri") - echo "KRUGS4ZANFZSA43PNVSSA43UOJUQ (is the answer)" - echo base32Encode("This is some str") - echo "KRUGS4ZANFZSA43PNVSSA43UOI (is the answer)" - - - echo "-----" - echo base32vEncode("This is some string.") - echo base32vDecode(base32vEncode("1his is some string.")) - echo base32vEncode("This is some string") - echo base32vDecode(base32vEncode("2his is some string")) - echo base32vEncode("This is some strin") - echo base32vDecode(base32vEncode("3his is some strin")) - echo base32vEncode("This is some stri") - echo base32vDecode(base32vEncode("4his is some stri")) - echo base32vEncode("This is some str") - echo base32vDecode(base32vEncode("5his is some str")) - - echo "-----" - echo base32Encode("This is some string.") - echo base32Decode(base32Encode("1his is some string.")) - echo base32Encode("This is some string") - echo base32Decode(base32Encode("2his is some string")) - echo base32Encode("This is some strin") - echo base32Decode(base32Encode("3his is some strin")) - echo base32Encode("This is some stri") - echo base32Decode(base32Encode("4his is some stri")) - echo base32Encode("This is some str") - echo base32Decode(base32Encode("5his is some str")) diff --git a/nimutils/file.nim b/nimutils/file.nim index a3b71fb..27e31b9 100644 --- a/nimutils/file.nim +++ b/nimutils/file.nim @@ -3,7 +3,75 @@ import os, posix, strutils, posix_utils -proc getMyAppPath(): string {.importc.} +when hostOs == "macosx": + {.emit: """ +#include +#include + + char *c_get_app_fname(char *buf) { + proc_pidpath(getpid(), buf, PROC_PIDPATHINFO_MAXSIZE); // 4096 + return buf; + } + """.} + + proc cGetAppFilename(x: cstring): cstring {.importc: "c_get_app_fname".} + + proc betterGetAppFileName(): string = + var x: array[4096, byte] + + return $(cGetAppFilename(cast[cstring](addr x[0]))) + +elif hostOs == "linux": + {.emit: """ +#include + + char *c_get_app_fname(char *buf) { + char proc_path[128]; + snprintf(proc_path, 128, "/proc/%d/exe", getpid()); + readlink(proc_path, buf, 4096); + return buf; + } + """.} + + proc cGetAppFilename(x: cstring): cstring {.importc: "c_get_app_fname".} + + proc betterGetAppFileName(): string = + var x: array[4096, byte] + + return $(cGetAppFilename(cast[cstring](addr x[0]))) +else: + template betterGetAppFileName(): string = getAppFileName() + +when hostOs == "macosx": + proc getMyAppPath*(): string {.exportc.} = + ## Returns the proper location of the running executable on disk, + ## resolving any file system links. + let name = betterGetAppFileName() + + if "_CHALK" notin name: + return name + let parts = name.split("_CHALK")[0 .. ^1] + + for item in parts: + if len(item) < 3: + return name + case item[0 ..< 3] + of "HM_": + result &= "#" + of "SP_": + result &= " " + of "SL_": + result &= "/" + else: + return name + if len(item) > 3: + result &= item[3 .. ^1] + echo "getMyAppPath() = ", result +else: + proc getMyAppPath*(): string {.exportc.} = + ## Returns the proper location of the running executable on disk, + ## resolving any file system links. + betterGetAppFileName() proc tildeExpand(s: string): string {.inline.} = var homedir = getHomeDir() @@ -23,7 +91,8 @@ proc tildeExpand(s: string): string {.inline.} = proc resolvePath*(inpath: string): string = ## This first does tilde expansion (e.g., ~/file or ~viega/file), ## and then normalizes the path, and expresses it as an absolute - ## path. The Nim os utilities don't do the tilde expansion. + ## path. The Nim os utilities don't do the tilde expansion, for + ## some unfathomable reasons. # First, resolve tildes, as Nim doesn't seem to have an API call to # do that for us. @@ -42,12 +111,16 @@ proc resolvePath*(inpath: string): string = return cur.normalizedPath().absolutePath() proc tryToLoadFile*(fname: string): string = + ## A wrapper around readFile that returns an empty string if a file + ## cannot be read. try: return readFile(fname) except: return "" proc tryToWriteFile*(fname: string, contents: string): bool = + ## A wrapper around writeFile that returns `true` if the file was + ## successfully written, and `false` otherwise. try: writeFile(fname, contents) return true @@ -55,6 +128,9 @@ proc tryToWriteFile*(fname: string, contents: string): bool = return false proc tryToCopyFile*(fname: string, dst: string): bool = + ## A wrapper around copyFile that returns `true` if the file was + ## successfully copied, and `false` otherwise. + try: copyFile(fname, dst) return true @@ -62,6 +138,11 @@ proc tryToCopyFile*(fname: string, dst: string): bool = return false template withWorkingDir*(dir: string, code: untyped) = + ## Changes the working directory of a process to the given + ## directory, thens runs a block of code. + ## + ## When the code block is exited in any way, the original working + ## directory is restored. let toRestore = getCurrentDir() @@ -80,21 +161,38 @@ const S_IXALL = S_IXUSR or S_IXGRP or S_IXOTH template isFile*(info: Stat): bool = + ## Test a posix stat object to see if it represents a regulat file. (info.st_mode and S_IFMT) == S_IFREG template hasUserExeBit*(info: Stat): bool = + ## Test a stat object to see if it's user-executable. + ## See `isExecutable()` for more complete testing. (info.st_mode and S_IXUSR) != 0 template hasGroupExeBit*(info: Stat): bool = + ## Test a stat object to see if it's group-executable. + ## See `isExecutable()` for more complete testing. (info.st_mode and S_IXGRP) != 0 template hasOtherExeBit*(info: Stat): bool = + ## Test a stat object to see if it's executable by others. + ## See `isExecutable()` for more complete testing. (info.st_mode and S_IXOTH) != 0 template hasAnyExeBit*(info: Stat): bool = + ## Test for any of the executable bits being set. + ## See `isExecutable()` for more complete testing. (info.st_mode and S_IXALL) != 0 proc isExecutable*(path: string): bool = + ## Tests to see if the current process has permissions to run the + ## file at the given location, and that the file is a valid + ## executable. + ## + ## Note that, since this operates on a path instead of a file + ## descriptor, there could be a TOCTOU bug. However, you'll learn + ## about that when you then try to execute, so not the end of the + ## world! try: let info = stat(path) @@ -126,6 +224,12 @@ proc isExecutable*(path: string): bool = proc findAllExePaths*(cmdName: string, extraPaths: seq[string] = @[], usePath = true): seq[string] = + ## This looks for valid executables of the given name that the + ## current process has permission to execute. Generally, when + ## multiple items are returned, you should want to run the first + ## returned item (which you can do via `findExePath()`). However, + ## this gives you the option to fall back on other executables if + ## something goes wrong (if they're present, of course). ## ## The priority here is to the passed command name, but if and only ## if it is a path; we're assuming that they want to try to run @@ -138,11 +242,11 @@ proc findAllExePaths*(cmdName: string, ## ## If all else fails, we search the PATH environment variable. ## - ## Note that we don't check for permissions problems (including - ## not-executable), and we do not open the file, so there's the - ## chance of the executable going away before we try to run it. + ## Note that we don't check for all possible issues that could cause + ## something not to run, and there's the chance of the executable + ## going away before we try to run it. ## - ## The point is, the caller should eanticipate failure. + ## The point is, the caller should anticipate failure. let (mydir, me) = getMyAppPath().splitPath() var @@ -164,6 +268,20 @@ proc findAllExePaths*(cmdName: string, if potential.isExecutable(): result.add(potential) +proc findExePath*(cmdName: string, + extraPaths: seq[string] = @[], + usePath = true): string = + ## This looks for valid executables of the given name that the + ## current process has permission to execute. It returns the first + ## matching executable, using the priority rulles described in + ## `findAllExePaths()`. + ## + ## If no executables are found, this returns the empty string. + + let options = cmdName.findAllExePaths(extraPaths, usePath) + if len(options) != 0: + return options[0] + {.emit: """ #include #include @@ -184,6 +302,8 @@ proc do_read_link(s: cstring, p: pointer): void {.cdecl,importc,nodecl.} proc get_path_max*(): cint {.cdecl,importc,nodecl.} proc readLink*(s: string): string = + ## A wrapper for the posix `readlink` call that also resolves any + ## relative paths in the result. var v = newStringOfCap(int(get_path_max())); do_read_link(cstring(s), addr s[0]) result = resolvePath(v) @@ -194,6 +314,9 @@ proc getAllFileNames*(dir: string, followFileLinks = false, yieldDirs = false, followDirLinks = false): seq[string] = + ## This is a slightly more sane API for scanning for file names than the + ## one provided in the nim standard API, primarily in that it is a single + ## consistent API whether you scan recursively or not. var kind: PathComponent if yieldFileLinks and followFileLinks: @@ -240,7 +363,7 @@ proc getAllFileNames*(dir: string, continue let fullpath = joinPath(dir, filename) var statbuf: Stat - if lstat(fullPath, statbuf) < 0: + if lstat(cstring(fullPath), statbuf) < 0: continue elif S_ISLNK(statbuf.st_mode): if dirExists(fullpath): diff --git a/nimutils/filetable.nim b/nimutils/filetable.nim index bea6225..d24637b 100644 --- a/nimutils/filetable.nim +++ b/nimutils/filetable.nim @@ -9,7 +9,7 @@ ## :Copyright: 2022, 2023, Crash Override, Inc. -import tables, strutils, os, system/nimscript +import tables, strutils, os type FileTable* = Table[string, string] @@ -17,13 +17,13 @@ type proc staticListFiles*(arg: string): seq[string] = - # Unfortunately, for whatever reason, system/nimutils's listFiles() - # doesn't seem to work from here, so we can't use listFiles(). As a - # result, we use a staticexec("ls") and parse. This obviously is - # not portable to all versions of Windows. - # - # This invocation of ls might not be super portable. Deserves a bit - # of testing. + ## Unfortunately, for whatever reason, system/nimutils's listFiles() + ## doesn't seem to work from here, so we can't use listFiles(). As a + ## result, we use a staticexec("ls") and parse. This obviously is + ## not portable to all versions of Windows. + ## + ## This is super hacky, but works well enough, without digging deep + ## into the Javascript runtime. result = @[] let @@ -36,6 +36,14 @@ proc staticListFiles*(arg: string): seq[string] = template newFileTable*(dir: static[string]): FileTable = + ## This will, at compile time, read files from the named directory, + ## and produce a `Table[string, string]` where the keys are the file + ## names (without path info), and the values are the file contents. + ## + ## This doesn't use the newer dictionary interface, and I think at + ## this point, our tooling is good enough that we don't need this + ## for our own uses, but no reason why it can't stay if others might + ## find it useful. var ret: FileTable = initTable[string, string]() path = instantiationInfo(fullPaths = true).filename.splitPath().head @@ -53,6 +61,7 @@ template newFileTable*(dir: static[string]): FileTable = ret template newOrderedFileTable*(dir: static[string]): OrderedFileTable = + ## Same as `newFileTable()` except uses an `OrderedTable`. var ret: OrderedFileTable = initOrderedTable[string, string]() path = instantiationInfo(fullPaths = true).filename.splitPath().head @@ -68,10 +77,3 @@ template newOrderedFileTable*(dir: static[string]): OrderedFileTable = ret[key] = fileContents ret - -when isMainModule: - const x = newFileTable("/Users/viega/dev/sami/src/help/") - - for k, v in x: - echo "Filename: ", k - echo "Contents: ", v[0 .. 40], "..." diff --git a/nimutils/hexdump.nim b/nimutils/hexdump.nim index 059ed26..324ece7 100644 --- a/nimutils/hexdump.nim +++ b/nimutils/hexdump.nim @@ -1,40 +1,53 @@ -# Wrapping a C implementation I've done in the past. Oddly, the main -# C function (now called chex) was always declared uint64_t, but the -# only way I could make Nim happy was if I changed them to `unsigned -# int`, which is okay on all the machines I care about anyway. -# -# :Author: John Viega (john@viega.org) +## :Author: John Viega (john@viega.org) +## Wrapping a C implementation I've done in the past. Oddly, the main +## C function (now called chex) was always declared uint64_t, but the +## only way I could make Nim happy was if I changed them to `unsigned +## int`, which is okay on all the machines I care about anyway. -import strutils, os, system/nimscript +import strutils, os static: - {.compile: joinPath(splitPath(currentSourcePath()).head, "hex.c").} + {.compile: joinPath(splitPath(currentSourcePath()).head, "c/hex.c").} proc hex*(s: string): string = + ## Like toHex() in strutils, but uses lowercase letters, as nature + ## intended. return s.toHex().toLowerAscii() -proc rawHexDump(x: pointer, sz: cuint, offset: cuint, width: cuint): +proc rawHexDump(x: pointer, sz: cuint, offset: uint64, width: cuint): cstring {.importc: "chex", cdecl.} -proc hexDump*(x: pointer, sz: uint, offset: uint = 0, width = 0): string = - # Hex dump memory from the - var tofree = rawHexDump(x, cuint(sz), cuint(offset), cuint(width)) +proc hexDump*(x: pointer, sz: uint, offset: int = -1, width = 0): string = + ## Produce a nice hex dump that is sized properly for the terminal + ## (or, alternately, sized to the width provided in the `width` + ## parameter). + ## + ## - The first parameter should be a memory address, generally taken + ## with `addr somevar` + ## - The second parameter is the number of bytes to dump. + ## - The third parameter indicates the start value for the offset + ## printed. The default is to use the pointer value, but you + ## can start it anywhere. + + + var + realOffset = if offset < 0: cast[uint64](x) else: uint64(offset) + tofree = rawHexDump(x, cuint(sz), realOffset, cuint(width)) result = $(tofree) dealloc(tofree) -proc strDump*(s: string): string = - result = hexDump(addr s[0], uint(len(s))) +template strAddr*(s: string): pointer = + ## Somewhere I wrote code that replicates the string's data structure, + ## and does the needed casting to get to this value, but both have a + ## potential race condition, so this is at least simple, though + ## it's identical to arrAddr here. + if s.len() == 0: nil else: addr s[0] -when isMainModule: - var - buf: array[128, byte] = [ - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, - 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, - 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, - 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, - 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, - 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, - 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, - 123, 124, 125, 126, 127 ] - - echo hexDump(addr buf[0], 128, width = 80) +template listAddr*[T](x: openarray[T]): pointer = + ## Return a pointer to an array or sequence, without crashing if the + ## array is empty. + if x.len() == 0: nil else: addr x[0] + +proc strDump*(s: string): string = + ## Produces a hex dump of the + result = hexDump(listAddr(s), uint(len(s))) diff --git a/nimutils/htmlparse.nim b/nimutils/htmlparse.nim index 3b168f8..995cbce 100644 --- a/nimutils/htmlparse.nim +++ b/nimutils/htmlparse.nim @@ -1,4 +1,5 @@ ## Wraps libgumbo for fast, standards compliant HTML parsing. +## Really only wraps the most basic functionality, though! ## ## :Author: John Viega (john@crashoverride.com) ## :Copyright: 2022 - 2023, Crash Override, Inc. @@ -65,15 +66,21 @@ proc add_attribute(ctx: var Walker, n, v: cstring) {.exportc, cdecl.} = ctx.cur.attrs[name] = val proc parseDocument*(html: string): HtmlNode = + ## Converts a string consisting of well-formed HTML into a tree + ## representing the DOM. + ## + ## If the string is not well-formed, results are undefined. That + ## means, since this call just uses the `gumbo` library to construct + ## the tree, we're not too well versed in the consequences of using + ## it with bunk input. We've gotten passable results, but with very + ## little experience here. var walker = Walker(root: nil, cur: nil) make_gumbo(cstring(html), cast[pointer](addr walker)) result = walker.root - -include "headers/gumbo.nim" - {.emit: """ +#include "gumbo.h" #include #include diff --git a/nimutils/logging.nim b/nimutils/logging.nim index e292266..840df9d 100644 --- a/nimutils/logging.nim +++ b/nimutils/logging.nim @@ -1,8 +1,7 @@ ## :Author: John Viega (john@crashoverride.com) ## :Copyright: 2023, Crash Override, Inc. -import tables, options, streams, pubsub, sinks, rope_construct, rope_ansirender, - strutils +import tables, options, pubsub, sinks, rope_base,rope_ansirender, rope_styles type LogLevel* = enum ## LogLevel describes what kind of messages you want to see. @@ -27,20 +26,12 @@ const llInfo: "info", llTrace: "trace" }.toTable() - pre = "" - post = "" - errPrefix = pre & "error: " & post - warnPrefix = pre & "warn: " & post - infoPrefix = pre & "info: " & post - trPrefix = pre & "trace: " & post - - var logLevelPrefixes = { llNone: "", - llError: errPrefix, - llWarn: warnPrefix, - llInfo: infoPrefix, - llTrace: trPrefix, + llError: $(defaultBg(fgColor("error: ", "red"))), + llWarn: $(defaultBg(fgColor("warn: ", "yellow"))), + llInfo: $(defaultBg(fgColor("info: ", "atomiclime"))), + llTrace: $(defaultBg(fgColor("trace: ", "jazzberry"))) }.toTable() const keyLogLevel* = "loglevel" @@ -49,29 +40,35 @@ var currentLogLevel = llInfo proc `$`*(ll: LogLevel): string = llToStrMap[ll] proc setLogLevelPrefix*(ll: LogLevel, prefix: string) = + ## Set the prefix used for messages of a given log level. logLevelPrefixes[ll] = prefix proc setLogLevel*(ll: LogLevel) = + ## Sets the current log level using values from the enum `LogLevel` currentLogLevel = ll proc setLogLevel*(ll: string) = + ## Sets the current log level using the english string. if ll in toLogLevelMap: setLogLevel(toLogLevelMap[ll]) else: raise newException(ValueError, "Invalid log level value: '" & ll & "'") -proc getLogLevel*(): LogLevel = currentLogLevel +proc getLogLevel*(): LogLevel = + ## Returns the current log level. + currentLogLevel proc logPrefixFilter*(msg: string, info: StringTable): (string, bool) = + ## A filter, installed by default, that adds a logging prefix to + ## the beginning of the message. if keyLogLevel in info: let llStr = info[keyLogLevel] if llStr in toLogLevelMap: let msgLevel = toLogLevelMap[llStr] - prefix = logLevelPrefixes[msgLevel].stylizeHtml(ensureNl = false) - return (prefix & msg, true) + return (logLevelPrefixes[msgLevel] & msg, true) else: raise newException(ValueError, "Log prefix filter used w/o passing in " & "a valid value for 'loglevel' in the publish() call's 'aux' " & @@ -80,11 +77,18 @@ proc logPrefixFilter*(msg: string, info: StringTable): (string, bool) = var suspendLogging = false proc toggleLoggingEnabled*() = + ## When logging is suspended, any published messages will be dropped when + ## filtering by log level. suspendLogging = not suspendLogging template getSuspendLogging*(): bool = suspendLogging proc logLevelFilter*(msg: string, info: StringTable): (string, bool) = + ## Filters out messages that are not important enough, per the currently + ## set log level. + ## + ## If toggleLoggingEnabled() has been called an odd number of times, + ## this filter will drop everything. if suspendLogging: return ("", false) if keyLogLevel in info: @@ -113,19 +117,27 @@ subscribe(logTopic, defaultLogHook) proc log*(level: LogLevel, msg: string) = + ## Generic interface for publishing messages at a given log level. discard publish(logTopic, msg & "\n", newOrderedTable({ keyLogLevel: llToStrMap[level] })) proc log*(level: string, msg: string) = + ## Generic interface for publishing messages at a given log level. discard publish(logTopic, msg & "\n", newOrderedTable({ keyLogLevel: level })) -proc error*(msg: string) = log(llError, msg) -proc warn*(msg: string) = log(llWarn, msg) -proc info*(msg: string) = log(llInfo, msg) -proc trace*(msg: string) = log(llTrace, msg) +template log*(level: LogLevel, msg: Rope) = log(level, $(msg)) +template log*(level: string, msg: Rope) = log(level, $(msg)) +template error*(msg: string) = log(llError, msg) +template warn*(msg: string) = log(llWarn, msg) +template info*(msg: string) = log(llInfo, msg) +template trace*(msg: string) = log(llTrace, msg) +template error*(msg: Rope) = log(llError, $(msg)) +template warn*(msg: Rope) = log(llWarn, $(msg)) +template info*(msg: Rope) = log(llInfo, $(msg)) +template trace*(msg: Rope) = log(llTrace, $(msg)) when not defined(release): let diff --git a/nimutils/macproc.nim b/nimutils/macproc.nim index b2a76f0..0a11a51 100644 --- a/nimutils/macproc.nim +++ b/nimutils/macproc.nim @@ -1,430 +1,135 @@ when not defined(macosx): - import macros static: error "macproc.nim only loads on macos" -{.emit: """ +import os, posix -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -typedef struct { - int id; - char *name; -} gidinfo_t; - -typedef struct { - int pid; - int uid; - int gid; - int euid; - int ppid; - char *username; - char *path; - int argc; - int envc; - char *memblock; - char **argv; - char **envp; - int numgroups; - gidinfo_t *gids; -} procinfo_t; - -procinfo_t *proc_list(size_t *count); -procinfo_t *proc_list_one(size_t *count, int pid); -void del_procinfo(procinfo_t *cur); - -extern int errno; -int maxalloc; - -void -del_procinfo(procinfo_t *cur) { - int i = 0; - - if (cur == NULL) { - return; - } - - while (cur[i].pid) { - if (cur[i].username != NULL) { - free(cur[i].username); - } - if (cur[i].path != NULL) { - free(cur[i].path); - } - free(cur[i].memblock); - - if (cur[i].numgroups != 0) { - for (int j = 0; j < cur[i].numgroups; j++) { - free(cur[i].gids[j].name); - } - free(cur[i].gids); - } - i++; - } - - free(cur); -} - -void __attribute__((constructor)) set_maxalloc() { - int kargmax[] = {CTL_KERN, KERN_ARGMAX}; - size_t size; - - if (sysctl(kargmax, 2, &maxalloc, &size, NULL, 0) == -1) { - abort(); - } -} - -void -get_more_procinfo(procinfo_t *cur, int pid) { - char *path = calloc(PROC_PIDPATHINFO_MAXSIZE, 1); - - proc_pidpath(pid, path, PROC_PIDPATHINFO_MAXSIZE); - cur->path = realloc(path, strlen(path) + 1); - - - int procargs[] = {CTL_KERN, KERN_PROCARGS2, pid}; - - size_t size; - - char *memblock = (char *)calloc(maxalloc, 1); - - if (!memblock) { - return; - } - - size = maxalloc; - - if (sysctl(procargs, 3, memblock, &size, NULL, 0) == -1) { - free(memblock); - cur->argc = 0; - cur->envc = 0; - return; - } - - memblock = realloc(memblock, size); - cur->argc = *(int *)memblock; - cur->memblock = memblock; - - char *p = memblock + sizeof(int); - - // Skip path info; it's only partial, which is why we use proc_pidpath() - while(*p != 0) { p++; } - - // Skip any nulls after the path; - while(*p == 0) { p++; } - - cur->argv = calloc(sizeof(char *), cur->argc); - - for (int i = 0; i < cur->argc; i++) { - cur->argv[i] = p; - - while (*p) p++; - p++; - } - - char *env_start = p; - - cur->envc = 0; - - while (*p) { - cur->envc++; - while(*p++); - } - - p = env_start; - cur->envp = calloc(sizeof(char *), cur->envc); - - for (int i = 0; i < cur->envc; i++) { - cur->envp[i] = p; - - while (*p) p++; - } -} - -/* Even though this seems like it allocates an insane amount of memory, - * It's still plenty fast. - * - * For instance, I get a len of 284472 (which I expect is the next pid?) - * but there are only 438 procs. - * - * The OS seems to put valid results all together, so the break - * statement appears to work fine. - * - * But I've tested performance w/ a continue in the second loop - * instead, and it's definitely a lot slower, but still runs in less - * than .1 sec on my laptop (as opposed to around .01 seconds). - */ -procinfo_t * -proc_list(size_t *count) { - int err; - struct kinfo_proc *result, *to_free; - procinfo_t *pi; - procinfo_t *to_return; - int name[] = { CTL_KERN, KERN_PROC, KERN_PROC_ALL}; - size_t i, len; - size_t valid = 0; - - // This loop should only ever run once from what I can tell, because - // the OS has us massively over-allocate. - while (true) { - err = sysctl(name, 3, NULL, &len, NULL, 0); - if (err != 0) { - return NULL; - } - - result = (struct kinfo_proc *)calloc(sizeof(struct kinfo_proc), len); - to_free = result; - - if (result == NULL) { - return NULL; - } - if (sysctl(name, 3, result, &len, NULL, 0) == -1) { - free(result); - } - else { - break; - } - } - - // Add an extra one where we drop in pid = 0 as a sentinel. - // Not that we're likely to need it. - pi = (procinfo_t *)calloc(sizeof(procinfo_t), len + 1); - to_return = pi; - - for (i = 0; i < len; i++) { - int pid = result->kp_proc.p_pid; - - if (!pid) { - printf("Stopping after: %d\n", i); - pi->pid = 0; - break; - } - - valid = valid + 1; - - pi->pid = pid; - pi->ppid = result->kp_eproc.e_ppid; - pi->uid = result->kp_eproc.e_pcred.p_ruid; - pi->gid = result->kp_eproc.e_pcred.p_rgid; - pi->euid = result->kp_eproc.e_ucred.cr_uid; - pi->numgroups = result->kp_eproc.e_ucred.cr_ngroups; - - struct passwd *pwent = getpwuid(pi->uid); - pi->username = strdup(pwent->pw_name); - - struct group *ginfo; - - if (pi->numgroups == 0) { - pi->gids = NULL; - } else { - // Seems to be a ton of dupes, so skipping them. - int sofar = 0; - pi->gids = calloc(sizeof(gidinfo_t), pi->numgroups); - - for (int i = 0; i < pi->numgroups; i++) { - for (int j = 0; j < sofar; j++) { - if(pi->gids[j].id == - result->kp_eproc.e_ucred.cr_groups[i]) { - goto skip_copy; - } - } - pi->gids[sofar].id = result->kp_eproc.e_ucred.cr_groups[i]; - ginfo = getgrgid(pi->gids[i].id); - pi->gids[sofar].name = strdup(ginfo->gr_name); - sofar++; - - skip_copy: - continue; - } - pi->numgroups = sofar; - } - - get_more_procinfo(pi, pid); - - pi++; - result++; - } - - free(to_free); - - *count = valid; - - to_return[valid].pid = 0; - - return realloc(to_return, sizeof(procinfo_t) * (valid + 1)); -} - -procinfo_t * -proc_list_one(size_t *count, int pid) { - int err; - struct kinfo_proc *result; - procinfo_t *to_return; - int name[] = { CTL_KERN, KERN_PROC, KERN_PROC_PID, pid}; - size_t i, len; - size_t valid = 0; - - while (true) { - err = sysctl(name, 4, NULL, &len, NULL, 0); - if (err != 0) { - return NULL; - } - - result = (struct kinfo_proc *)calloc(sizeof(struct kinfo_proc), len); - if (result == NULL) { - return NULL; - } - if (sysctl(name, 4, result, &len, NULL, 0) == -1) { - free(result); - } - else { - if (len != 1) { - return NULL; - } - break; - } - } - - to_return = (procinfo_t *)calloc(sizeof(procinfo_t), len); - - for (i = 0; i < len; i++) { - struct kinfo_proc *oneProc = &result[i]; - int pid = oneProc->kp_proc.p_pid; - - if (!pid) continue; - - valid = valid + 1; - - to_return[i].pid = pid; - to_return[i].ppid = oneProc->kp_eproc.e_ppid; - to_return[i].uid = oneProc->kp_eproc.e_ucred.cr_uid; - - get_more_procinfo(&to_return[i], pid); - } - - free(result); - - *count = valid; - - return to_return; -} - -#if 1 -int -demo_ps() { - procinfo_t *info; - size_t num; - int err; - - info = proc_list(&num); - - for (int i = 0; i < num; i++) { - printf("%6d: %s ", info[i].pid, info[i].path); - for(int j = 0; j < info[i].argc; j++) { - printf("%s ", info[i].argv[j]); - } - printf("uid = %d gid = %d ppid = %d uname = %s nargs = %d nenv = %d", - info[i].uid, info[i].gid, info[i].ppid, - info[i].username, info[i].argc, info[i].envc); - - if (info[i].numgroups != 0) { - printf(" #groups = %d groups = ", info[i].numgroups); - for (int j = 0; j < info[i].numgroups; j++) { - printf("%s(%d) ", info[i].gids[j].name, - info[i].gids[j].id); - } - } - - printf("\n"); - } - - printf("\nFound %zu procs\n", num); - - del_procinfo(info); - return 0; -} -#endif - -""".} - -from macros import hint +{.compile: joinPath(splitPath(currentSourcePath()).head, "c/macproc.c").} type - gidinfot_469762363 {.pure, inheritable, bycopy.} = object - id*: cint ## Generated based on /Users/viega/dev/chalk-internal/futhark/macproc.h:16:9 + CGidInfo* {.importc: "gidinfo_t", header: "macproc.h", bycopy.} = object + id*: cint name*: cstring - procinfot_469762366 {.pure, inheritable, bycopy.} = object - pid*: cint ## Generated based on /Users/viega/dev/chalk-internal/futhark/macproc.h:21:9 - uid*: cint - gid*: cint - euid*: cint - ppid*: cint - username*: cstring - path*: cstring - argc*: cint - envc*: cint - memblock*: cstring - argv*: ptr ptr cschar - envp*: ptr ptr cschar + CProcInfo* {.importc: "procinfo_t", header: "macproc.h", bycopy.} = object + pid*: cint + uid*: cint + gid*: cint + euid*: cint + ppid*: cint + username*: cstring + path*: cstring + argc*: cint + envc*: cint + memblock*: cstring + argv*: ptr UncheckedArray[cstring] + envp*: ptr UncheckedArray[cstring] numgroups*: cint - gids*: ptr gidinfot_469762365 - - procinfot_469762367 = (when declared(procinfot): - procinfot - else: - procinfot_469762366) - gidinfot_469762365 = (when declared(gidinfot): - gidinfot - else: - gidinfot_469762363) - -when not declared(procinfot): - type - procinfot* = procinfot_469762366 -else: - static : - hint("Declaration of " & "procinfot" & " already exists, not redeclaring") -when not declared(gidinfot): - type - gidinfot* = gidinfot_469762363 -else: - static : - hint("Declaration of " & "gidinfot" & " already exists, not redeclaring") -when not declared(proclist): - proc proclist*(count: ptr csize_t): ptr procinfot_469762367 {.cdecl, - importc: "proc_list".} -else: - static : - hint("Declaration of " & "proclist" & " already exists, not redeclaring") -when not declared(proclistone): - proc proclistone*(count: ptr csize_t; pid: cint): ptr procinfot_469762367 {. - cdecl, importc: "proc_list_one".} -else: - static : - hint("Declaration of " & "proclistone" & " already exists, not redeclaring") -when not declared(delprocinfo): - proc delprocinfo*(cur: ptr procinfot_469762367): void {.cdecl, - importc: "del_procinfo".} -else: - static : - hint("Declaration of " & "delprocinfo" & " already exists, not redeclaring") - - -proc demops*(): cint {.discardable,cdecl,importc: "demo_ps".} - -type ProcInfoT = object of procInfoT -proc `=destroy`*(ctx: var ProcInfoT) = - delProcInfo(addr ctx) - -when isMainModule: - demops() + gids*: ptr UncheckedArray[CGidInfo] + + GroupInfo* = object + id*: Gid + name*: string + + ProcessInfo* = object + pid*: Pid + uid*: Uid + gid*: Gid + euid*: Uid + ppid*: Pid + username*: string + path*: string + argv*: seq[string] + envp*: seq[string] + groups*: seq[GroupInfo] + +proc proc_list*(count: ptr csize_t): ptr UncheckedArray[CProcInfo] + {. cdecl, importc, header: "macproc.h" .} +proc proc_list_one*(count: ptr csize_t; pid: Pid): ptr CProcInfo + {. cdecl, importc, header: "macproc.h" .} +proc del_procinfo*(cur: ptr CProcInfo) {. cdecl, importc, header: "macproc.h" .} + +template copyProcessInfo(nimOne: var ProcessInfo, cOne: CProcInfo) = + nimOne.pid = Pid(cOne.pid) + nimOne.uid = Uid(cOne.uid) + nimOne.gid = Gid(cOne.gid) + nimOne.euid = Uid(cOne.euid) + nimOne.ppid = Pid(cOne.ppid) + nimOne.username = $(cOne.username) + nimOne.path = $(cOne.path) + + if cOne.argc > 0: + for j in 0 ..< cOne.argc: + nimOne.argv.add($(cOne.argv[j])) + + if cOne.envc > 0: + for j in 0 ..< cOne.envc: + nimOne.envp.add($(cOne.envp[j])) + + if cOne.numgroups > 0: + for j in 0 ..< cOne.numgroups: + let grpObj = cOne.gids[j] + nimOne.groups.add(GroupInfo(id: Gid(grpObj.id), name: $(grpObj.name))) + +proc listProcesses*(): seq[ProcessInfo] = + ## Return process info for all processes on a MacOs machine. + var num: csize_t + + let procInfo = proc_list(addr num) + + for i in 0 ..< num: + var nimOne: ProcessInfo + + nimOne.copyProcessInfo(procInfo[i]) + result.add(nimOne) + + del_procinfo(addr procInfo[0]) + +proc getProcessInfo*(pid: Pid): ProcessInfo = + ## Return process info for a given process on a MacOs machine. + var num: csize_t + + let procInfo = proc_list_one(addr num, pid) + + if num == 0: + raise newException(ValueError, "PID not found") + + result.copyProcessInfo(procInfo[]) + del_procinfo(procInfo); + +proc getPid*(o: ProcessInfo): Pid = + ## Returns the process ID. + o.pid +proc getUid*(o: ProcessInfo): Uid = + ## Returns the user ID of the process. + o.uid +proc getGid*(o: ProcessInfo): Gid = + ## Returns the primary group ID of the process. + o.gid +proc getEuid*(o: ProcessInfo): Uid = + ## Returns the effective uid for the process. + o.euid +proc getParentPid*(o: ProcessInfo): Pid = + ## Returns the PID of the parent process. + o.ppid +proc getUserName*(o: ProcessInfo): string = + ## Returns the user name associated with the process' UID + o.username +proc getExePath*(o: ProcessInfo): string = + ## Returns the file system path for the executable. + o.path +proc getArgv*(o: ProcessInfo): seq[string] = + ## Returns the value of the program's argv + o.argv +proc getEnvp*(o: ProcessInfo): seq[string] = + ## Returns the contents of the environment passed to the process. + o.envp +proc getGroups*(o: ProcessInfo): seq[GroupInfo] = + ## Any auxillary groups the process is in. + o.groups +proc getGid*(o: GroupInfo): Gid = + ## Returns a GID + o.id +proc getGroupName*(o: GroupInfo): string = + ## Returns the group name associated with a process. + o.name diff --git a/nimutils/managedtmp.nim b/nimutils/managedtmp.nim index 1c551bb..6af72d8 100644 --- a/nimutils/managedtmp.nim +++ b/nimutils/managedtmp.nim @@ -15,16 +15,29 @@ var proc getNewTempDir*(tmpFilePrefix = defaultTmpPrefix, tmpFileSuffix = defaultTmpSuffix): string = + ## Returns a new temporary directory that is `managed`, meaning it + ## will be automatically removed when the program exits. + ## + ## Note that this only applies for normal exits; you will generally + ## need to register a signal handler to call `tmpfileOnExit()` + ## if there is an abnormal exit. result = createTempDir(tmpFilePrefix, tmpFileSuffix) managedTmpDirs.add(result) proc getNewTempFile*(prefix = defaultTmpPrefix, suffix = defaultTmpSuffix, autoClean = true): (FileStream, string) = + ## Returns a new temporary file that is `managed`, meaning it + ## will be automatically removed when the program exits. + ## + ## Note that this only applies for normal exits; you will generally + ## need to register a signal handler to call `tmpfileOnExit()` + ## if there is an abnormal exit. + # in some cases such as docker, due to snap permissions # it does not have access directly to files created in /tmp # but it can access those files if they are nested in another - # nested dir + # directory. let dir = genTempPath(prefix, suffix) createDir(dir) var (f, path) = createTempFile(prefix, suffix, dir = dir) @@ -34,21 +47,31 @@ proc getNewTempFile*(prefix = defaultTmpPrefix, result = (newFileStream(f), path) template registerTempFile*(path: string) = + ## Register a managed temp file created via some other interface. managedTmpFiles.add(path) template registerTmpDir*(path: string) = + ## Register a managed temp directory created via some other interface. managedTmpDirs.add(path) template setManagedTmpExitCallback*(cb: OnExitTmpFileCallback) = + ## If you add a callback, you can report on any errors in deleting, or, + ## if temp files are to be moved instead of deleted, you can report + ## on what's been saved where. exitCallback = cb template setManagedTmpCopyLocation*(loc: string) = + ## If this path is set, temp files will not be copied, they will + ## instead be moved to a directory under the given location. + ## Report on it with an exit callback. onExitCopyDir = loc template setDefaultTmpFilePrefix*(s: string) = + ## Set the default prefix to use for created temp files. defaultTmpPrefix = s template setDefaultTmpFileSuffix*(s: string) = + ## Set the default suffix to use for created temp files. defaultTmpSuffix = s {.pragma: destructor, @@ -57,6 +80,11 @@ template setDefaultTmpFileSuffix*(s: string) = # codegenDecl: "__attribute__((constructor)) $# $#$#", exportc.} proc tmpfile_on_exit*() {.destructor.} = + ## This will get called automatically on normal program exit, but + ## must be called manually if terminating due to a signal. + ## + ## It implements the logic for deleting, moving and calling any + ## callback. var fileList: seq[string] dirList: seq[string] diff --git a/nimutils/markdown.nim b/nimutils/markdown.nim index 907c22d..970eb04 100644 --- a/nimutils/markdown.nim +++ b/nimutils/markdown.nim @@ -3,40 +3,45 @@ ## :Author: John Viega (john@crashoverride.com) ## :Copyright: 2022 - 2023, Crash Override, Inc. -include "headers/md4c.nim" + + import random -type MdOpts* = enum - MdCommonMark = 0x00000000, - MdCollapseWhiteSpace = 0x00000001, - MdPermissiveAtxHeaders = 0x00000002, - MdPermissiveUrlAutoLinks = 0x00000004, - MdPermissiveMailAutoLinks = 0x00000008, - MdNoIndentedCodeBlocks = 0x00000010, - MdNoHtmlBlocks = 0x00000020, - MdNoHtmlpans = 0x00000040, - MdNoHtml = 0x00000060, - MdTables = 0x00000100, - MdStrikeThrough = 0x00000200, - MdPermissiveWwwAutoLinks = 0x00000400, - MdPermissiveAutoLinks = 0x0000040c, - MdTaskLists = 0x00000800, - MdLatexMathSpans = 0x00001000, - MdWikiLinks = 0x00002000, - MdUnderline = 0x00004000, - MdHeaderSelfLinks = 0x00008000, - MdGithub = 0x00008f0c, - MdCodeLinks = 0x00010000, - MdHtmlDebugOut = 0x10000000, - MdHtmlVerbatimEntries = 0x20000000, - MdHtmlSkipBom = 0x40000000, - MdHtmlXhtml = 0x80000000 +{.emit: """#include "md4c.h" """.} + +type + # We don't actually use this type, just pulls in the header concisely. + MdOpts* = enum + MdCommonMark = 0x00000000, + MdCollapseWhiteSpace = 0x00000001, + MdPermissiveAtxHeaders = 0x00000002, + MdPermissiveUrlAutoLinks = 0x00000004, + MdPermissiveMailAutoLinks = 0x00000008, + MdNoIndentedCodeBlocks = 0x00000010, + MdNoHtmlBlocks = 0x00000020, + MdNoHtmlpans = 0x00000040, + MdNoHtml = 0x00000060, + MdTables = 0x00000100, + MdStrikeThrough = 0x00000200, + MdPermissiveWwwAutoLinks = 0x00000400, + MdPermissiveAutoLinks = 0x0000040c, + MdTaskLists = 0x00000800, + MdLatexMathSpans = 0x00001000, + MdWikiLinks = 0x00002000, + MdUnderline = 0x00004000, + MdHeaderSelfLinks = 0x00008000, + MdGithub = 0x00008f0c, + MdCodeLinks = 0x00010000, + MdHtmlDebugOut = 0x10000000, + MdHtmlVerbatimEntries = 0x20000000, + MdHtmlSkipBom = 0x40000000, + MdHtmlXhtml = 0x80000000 type HtmlOutputContainer = ref object s: string -proc nimu_process_markdown(s: ptr UncheckedArray[char], n: cuint, p: pointer) {.cdecl,exportc.} = +proc nimu_process_markdown(s: ptr UncheckedArray[byte], n: cuint, p: pointer) {.cdecl, exportc.} = var x: HtmlOutputContainer = (cast[ptr HtmlOutputContainer](p))[] x.s.add(bytesToString(s, int(n))) @@ -45,6 +50,9 @@ proc c_markdown_to_html(s: cstring, l: cuint, o: pointer, f: cint): cint {.importc, cdecl,nodecl.} proc markdownToHtml*(s: string, opts: openarray[MdOpts] = [MdGithub]): string = + ## Converts a string from Markdown to an html string. The string can + ## have embedded markdown. This functionality is implemented via the + ## MD4C library. var container = HtmlOutputContainer() res: cint @@ -58,12 +66,3 @@ proc markdownToHtml*(s: string, opts: openarray[MdOpts] = [MdGithub]): string = # First removing the good stuff makes it easier to replace reliably. result = container.s - -when isMainModule: - echo markdownToHtml(""" -# Hello world! - -| Example | Table | -| ------- | ----- | -| foo | bar | -""") diff --git a/nimutils/misc.nim b/nimutils/misc.nim index c8c84ba..7439112 100644 --- a/nimutils/misc.nim +++ b/nimutils/misc.nim @@ -102,7 +102,22 @@ proc copy*[T](data: sink T): ref T = proc getpass*(prompt: cstring) : cstring {.header: "", header: "", importc: "getpass".} + ## The raw getpass function. Nim 2.0 now wraps this, so could be + ## deprecated. + +template getPassword*(prompt: string): string = + ## Retrieve a password from the terminal (turning off echo for the + ## duration of the input). This is now part of Nim 2, but whatever. + $(getpass(cstring(prompt))) proc bytesToString*(bytes: openarray[byte]): string = + ## Take raw bytes and copy them into a string object. result = newString(bytes.len) copyMem(result[0].addr, bytes[0].unsafeAddr, bytes.len) + +proc bytesToString*(bytes: pointer, l: int): string = + ## Converts bytes at a memory address to a string. + if bytes == nil: + return "" + result = newString(l) + copyMem(result[0].addr, bytes, l) diff --git a/nimutils/net.nim b/nimutils/net.nim index c0a0620..f0aafae 100644 --- a/nimutils/net.nim +++ b/nimutils/net.nim @@ -38,5 +38,8 @@ get_external_ipv4_address() proc get_external_ipv4_address() : cstring {.cdecl, importc.} proc getMyIpV4Addr*(): string = + ## Portably returns the primary IPv4 address as determined by the + ## machine's routing table. However, this does require internet + ## access. var s = get_external_ipv4_address() result = $(s) diff --git a/nimutils/nimscript.nim b/nimutils/nimscript.nim new file mode 100644 index 0000000..f7e70d5 --- /dev/null +++ b/nimutils/nimscript.nim @@ -0,0 +1,104 @@ +import os, strutils +export os, strutils + +var + targetArch* = hostCPU + targetStr*: string + + +proc setTargetStr(target: string) = + targetStr = target + +proc setupTargetArch(quiet = true) = + once: + when defined(macosx): + # -d:arch=amd64 will allow you to specifically cross-compile to intel. + # The .strdefine. pragma sets the variable from the -d: flag w/ the same + # name, overriding the value of the const. + const arch {.strdefine.} = "detect" + + var + targetStr = "" + + if arch == "detect": + # On an x86 mac, the proc_translated OID doesn't exist. So if this + # returns either 0 or 1, we know we're running on an arm. Right now, + # nim will always use rosetta, so should always give us a '1', but + # that might change in the future. + let sysctlOut = staticExec("sysctl -n sysctl.proc_translated") + + if sysctlOut in ["0", "1"]: + targetArch = "arm64" + else: + targetArch = "amd64" + else: + echo "Override: arch = " & arch + + if targetArch == "arm64": + if not quiet: + echo "Building for arm64" + setTargetStr("arm64-apple-macos13") + elif targetArch == "amd64": + setTargetStr("x86_64-apple-macos13") + if not quiet: + echo "Building for amd64" + else: + if not quiet: + echo "Invalid target architecture for MacOs: " & arch + quit(1) + +proc getTargetArch*(): string = + ## The Nim compile time runs in the Javascript VM. On a Mac, for + ## whatever crazy reason, the VM runs in an X86 emulator, meaning + ## that Nim's `hostCPU` builtin will always report `amd64`, even when + ## it should be reporting `arm` on M1/2/3 macs. + ## + ## This uses some trickery to detect when the underlying machine is + ## `arm`. If you set -d:arch=amd64 it will override. + ## + ## Meant to be run from your config.nims file. + + setupTargetArch() + return targetArch + +template applyCommonLinkOptions*(staticLink = true, quiet = true) = + ## Applies the link options necessary for projects using nimutils. + ## Meant to be called from your config.nims file. + switch("d", "ssl") + switch("d", "nimPreviewHashRef") + switch("gc", "refc") + switch("path", ".") + switch("d", "useOpenSSL3") + switch("cincludes", getEnv("HOME").joinPath("/.local/c0/include")) + + setupTargetArch(quiet) + + when defined(macosx): + switch("cpu", targetArch) + switch("passc", "-flto -target " & targetStr) + switch("passl", "-flto -w -target " & targetStr & + "-Wl,-object_path_lto,lto.o") + elif defined(linux): + if staticLink: + switch("passc", "-static") + switch("passl", "-static") + else: + discard + else: + echo "Platform not supported." + quit(1) + +template staticLinkLibraries*(libNames: openarray[string], libDir: string, + useMusl = true, muslBase = libDir) = + ## Automates statically linking all appropriate libraries. + ## Meant to be called from your config.nims file. + when defined(linux): + if useMusl: + let muslPath = muslBase & "musl/bin/musl-gcc" + switch("gcc.exe", muslPath) + switch("gcc.linkerexe", muslPath) + + for item in libNames: + let libFile = "lib" & item & ".a" + switch("passL", libDir.joinPath(libFile)) + switch("dynlibOverride", item) diff --git a/nimutils/process.nim b/nimutils/process.nim deleted file mode 100644 index 55daf59..0000000 --- a/nimutils/process.nim +++ /dev/null @@ -1,90 +0,0 @@ -## Process-related utilities. -## -## :Author: John Viega (john@crashoverride.com) -## :Copyright: 2023, Crash Override, Inc. - -import misc, strutils, posix, os, options - -when hostOs == "macosx": - {.emit: """ -#include -#include - - char *c_get_app_fname(char *buf) { - proc_pidpath(getpid(), buf, PROC_PIDPATHINFO_MAXSIZE); // 4096 - return buf; - } - """.} - - proc cGetAppFilename(x: cstring): cstring {.importc: "c_get_app_fname".} - - proc betterGetAppFileName(): string = - var x: array[4096, byte] - - return $(cGetAppFilename(cast[cstring](addr x[0]))) - -elif hostOs == "linux": - {.emit: """ -#include - - char *c_get_app_fname(char *buf) { - char proc_path[128]; - snprintf(proc_path, 128, "/proc/%d/exe", getpid()); - readlink(proc_path, buf, 4096); - return buf; - } - """.} - - proc cGetAppFilename(x: cstring): cstring {.importc: "c_get_app_fname".} - - proc betterGetAppFileName(): string = - var x: array[4096, byte] - - return $(cGetAppFilename(cast[cstring](addr x[0]))) -else: - template betterGetAppFileName(): string = getAppFileName() - -when hostOs == "macosx": - proc getMyAppPath*(): string {.exportc.} = - let name = betterGetAppFileName() - - if "_CHALK" notin name: - return name - let parts = name.split("_CHALK")[0 .. ^1] - - for item in parts: - if len(item) < 3: - return name - case item[0 ..< 3] - of "HM_": - result &= "#" - of "SP_": - result &= " " - of "SL_": - result &= "/" - else: - return name - if len(item) > 3: - result &= item[3 .. ^1] - echo "getMyAppPath() = ", result -else: - proc getMyAppPath*(): string {.exportc.} = betterGetAppFileName() - -proc getPasswordViaTty*(): string {.discardable.} = - if isatty(0) == 0: - return "" - - var pw = getpass(cstring("Enter password for decrypting the private key: ")) - - result = $(pw) - - for i in 0 ..< len(pw): - pw[i] = char(0) - -proc delByValue*[T](s: var seq[T], x: T): bool {.discardable.} = - let ix = s.find(x) - if ix == -1: - return false - - s.delete(ix) - return true diff --git a/nimutils/progress.nim b/nimutils/progress.nim index d77b91d..1974339 100644 --- a/nimutils/progress.nim +++ b/nimutils/progress.nim @@ -45,6 +45,8 @@ template updateBar(txt: string) = stdout.flushFile() proc update*(ctx: var ProgressBar, newCur: int): bool {.discardable.} = + ## Update the number of items completed, and, if needed, redraw the + ## progress bar. var redraw = false if newCur == ctx.curItems: @@ -116,6 +118,14 @@ proc initProgress*(ctx: var ProgressBar, totalItems: int, showTime = true, curChar = Rune('>'), timeColor = "atomiclime", progColor = "jazzberry", curColor = "jazzberry", pctColor = "atomiclime", barColor = "jazzberry") = + ## Initializes a basic progress bar "widget", which runs on the + ## command line. + ## + ## `totalItems` is in whatever units you wish; call `update` to + ## redraw the bar if needed, where you will pass the number of items + ## you've completed towards the total. + ## + ## This widget assumes you do no other IO. hideCursor() ctx.totalItems = totalItems diff --git a/nimutils/prp.nim b/nimutils/prp.nim index 63a1300..f7c9923 100644 --- a/nimutils/prp.nim +++ b/nimutils/prp.nim @@ -1,5 +1,3 @@ -import aes, sha, random, hexdump - ## We're going to make a PRP using the Luby-Rackoff construction. The ## easiest thing for us to do is to break the input into two 'halves', ## one being 128 bits (the width of AES, which we will call the 'left @@ -38,8 +36,11 @@ import aes, sha, random, hexdump ## ## PRPs are reversable, and with feistel contstructions, it's by ## running the rounds backward. But instead of calling them 'encrypt' -## and 'decrypt', we use reversed names... on the horizontal axis. -## `brb` seems like a good function name for encryption. +## and 'decrypt', we use reversed (horizontally) and mirrored names... +## `brb` seems like a good function name for decryption. + +import aes, sha, random + type PrpCtx = object contents: string @@ -79,7 +80,7 @@ proc xor_in_place(o: pointer, s: cstring, i: cint): proc runHmacPrf(ctx: PrpCtx, key: string) = var toXor = key.hmacSha3(ctx.contents[16..^1]) - xor_in_place(addr ctx.contents[0], toXor, cint(16)) + xor_in_place(addr ctx.contents[0], cstring(toXor), cint(16)) proc runCtrPrf(ctx: PrpCtx, key: string) = aesCtrInPlaceOneShot(key, addr ctx.contents[16], cint(len(ctx.contents) - 16)) @@ -98,6 +99,23 @@ template round4(ctx: PrpCtx) = proc prp*(key, toEncrypt: string, nonce: var string, randomNonce = true): string = + ## Implements a 4-round Luby Rackoff PRP, which accepts inputs to + ## permute of 24 bytes or more. + ## + ## As long as we do not duplicate messages, this function will allow + ## us to do authenticated encryption without message expansion, with + ## no issues with nonce reuse, or bit-flipping attacks. + ## + ## This function is intended more for encrypted storage, where it's + ## a better option for most use cases than something based on + ## GCM. + ## + ## The practical downsides are: + ## + ## 1. It doesn't support streaming, so the whole message needs to be + ## in memory. + ## 2. It uses more crypto operations, and rekeys AES more. + ## 3. The fact that we didn't bother tweak for small messages. if toEncrypt.len() < 24: raise newException(ValueError, "Minimum supported length for " & @@ -119,6 +137,8 @@ proc prp*(key, toEncrypt: string, nonce: var string, randomNonce = true): return $(ctx.contents) proc brb*(key, toDecrypt: string, nonce: string): string = + ## The reverse permutation for our 4-round Luby Rackoff PRP. + if toDecrypt.len() < 24: raise newException(ValueError, "Minimum supported length for " & "messages encrypted with our PRP is 24 bytes") @@ -130,16 +150,3 @@ proc brb*(key, toDecrypt: string, nonce: string): string = ctx.round1() return ctx.contents - - -when isMainModule: - var - nonce: string - key = "0123456789abcdef" - ct = prp(key, - "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" & - "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", - nonce) - - echo ct.hex() - echo brb(key, ct, nonce) diff --git a/nimutils/pubsub.nim b/nimutils/pubsub.nim index 0e5fabb..20e3168 100644 --- a/nimutils/pubsub.nim +++ b/nimutils/pubsub.nim @@ -2,6 +2,9 @@ ## :Copyright: 2023, Crash Override, Inc. ## For now, not intended to be threadsafe for mutation ops. +## Additionally, this is some of the oldest code in Nimutils; it +## probably should be updated / merged with the lower level IO +## multiplexing switchboard that is now part of the library. import tables, sugar, options, json, strutils, strutils, std/terminal, unicodeid, rope_ansirender, rope_construct diff --git a/nimutils/random.nim b/nimutils/random.nim index 0b617db..68968ec 100644 --- a/nimutils/random.nim +++ b/nimutils/random.nim @@ -1,7 +1,10 @@ ## :Author: John Viega (john@crashoverride.com) ## :Copyright: 2022, 2023, Crash Override, Inc. -import std/sysrand, openssl, strutils +import std/sysrand, openssl, misc + +# This used to be in here. +export bytesToString template secureRand*[T](): T = ## Returns a uniformly distributed random value of any _sized_ type without @@ -13,6 +16,8 @@ template secureRand*[T](): T = cast[T](randBytes) proc randInt*(): int = + ## Returns a uniformly distributed integer across 63 bits... meaning + ## it is guaranteed to be positive. return secureRand[int] and (not (1 shl 63)) # I'm hitting what seems to be a nim 2.0 bug w/ urandom() to a @@ -22,28 +27,10 @@ proc randInt*(): int = proc RAND_bytes(p: pointer, i: cint): cint {.cdecl, dynlib: DLLUtilName, importc.} -proc bytesToString*(b: ptr UncheckedArray[char], l: int): string = - for i in 0 ..< l: - result.add(b[i]) - -template bytesToString*(b: ptr char, l: int): string = - bytesToString(cast[ptr UncheckedArray[char]](b), l) - proc randString*(l: int): string = ## Get a random binary string of a particular length. - var b = cast[ptr char](alloc(l)) + var b = cast[pointer](alloc(l)) discard RAND_bytes(b, cint(l)) - result = bytesToString(cast[ptr UncheckedArray[char]](b), l) + result = bytesToString(b, l) dealloc(b) - -template randStringHex*(l: int): string = - randString(l).toHex().toLowerAscii() - -when isMainModule: - import strutils - echo secureRand[uint64]() - echo secureRand[int32]() - echo secureRand[float]() - echo secureRand[array[6, byte]]() - echo randString(12).toHex() diff --git a/nimutils/rope_ansirender.nim b/nimutils/rope_ansirender.nim index 2bc0d0f..ac38ae0 100644 --- a/nimutils/rope_ansirender.nim +++ b/nimutils/rope_ansirender.nim @@ -70,26 +70,52 @@ proc ansiStyleInfo(b: TextPlane, ch: uint32): AnsiStyleInfo = if len(codes) > 0: result.ansiStart = "\e[" & codes.join(";") & "m" + else: + result.ansiStart = "\e[0m" + +template canColor(): bool = + if noColor or foundNoColor or not getShowColor(): + false + else: + true -proc preRenderBoxToAnsiString*(b: TextPlane, ensureNl = true): string = +proc preRenderBoxToAnsiString*(b: TextPlane, noColor = false): string = + ## Low-level interface for taking our lowest-level internal + ## representation, where the exact layout of the output is fully + ## specified, and converting it into ansi codes for the terminal. # TODO: Add back in unicode underline, etc. var styleInfo: AnsiStyleInfo - shouldTitle = false + colorStack: seq[bool] + shouldTitle = false + foundNoColor = false for line in b.lines: - for ch in line: + for i, ch in line: if ch > 0x10ffff: if ch == StylePop: - continue + if canColor(): + result &= ansiReset() + elif ch == StyleColor: + colorStack.add(foundNoColor) + foundNoColor = false + elif ch == StyleNoColor: + colorStack.add(foundNoColor) + foundNoColor = true + elif ch == StyleColorPop: + if len(colorStack) != 0: + foundNoColor = colorStack.pop() else: styleInfo = b.ansiStyleInfo(ch) - if styleInfo.ansiStart.len() > 0 and getShowColor(): + if canColor(): result &= ansiReset() result &= styleInfo.ansiStart if styleInfo.casing == CasingTitle: shouldTitle = true else: + if ch == uint32('\e'): + raise newException(ValueError, "ANSI escape codes are not allowed " & + "in text in this API") case styleInfo.casing of CasingTitle: if Rune(ch).isAlpha(): @@ -110,35 +136,69 @@ proc preRenderBoxToAnsiString*(b: TextPlane, ensureNl = true): string = result &= $(Rune(ch)) if not b.softBreak: result &= "\n" - if getShowColor(): + if canColor(): result &= ansiReset() - # if ensureNl and not result.endswith("\n"): - # if styleInfo.ansiStart.len() > 0 and getShowColor(): - # result &= ansiReset() & styleInfo.ansiStart & "\n" & ansiReset() - # else: - # result = "\n" +template render(r: Rope, width: int, showLinks: bool, style: FmtStyle, + ensureNl: bool, noColor: bool, outerPad: bool): string = + var toRender: Rope + if ensureNl and r.noBoxRequired(): + toRender = ensureNewline(r) + else: + toRender = r + toRender.preRender(width, showLinks, style, outerPad). + preRenderBoxToAnsiString(noColor) -template stylizeMd*(s: string, width = -1, showLinks = false, - ensureNl = true, style = defaultStyle): string = +template stylizeMd*(s: string, width = 0, showLinks = false, + ensureNl = true, style = defaultStyle, + noColor = false, outerPad = true): string = s.htmlStringToRope(). - preRender(width, showLinks, style). - preRenderBoxToAnsiString(ensureNl) + render(width, showLinks, style, ensureNl, noColor, outerPad) -template stylizeHtml*(s: string, width = -1, showLinks = false, - ensureNl = true, style = defaultStyle): string = +template stylizeHtml*(s: string, width = 0, showLinks = false, + ensureNl = true, style = defaultStyle, + noColor = false, outerPad = true): string = s.htmlStringToRope(false). - preRender(width, showLinks, style). - preRenderBoxToAnsiString(ensureNl) + render(width, showLinks, style, ensureNl, noColor, outerPad) -proc stylize*(s: string, width = -1, showLinks = false, - ensureNl = true, style = defaultStyle): string = - let r = Rope(kind: RopeAtom, text: s.toRunes()) - return r.preRender(width, showLinks, style). - preRenderBoxToAnsiString(ensureNl) +proc stylize*(s: string, width = -1, showLinks = false, ensureNl = true, + style = defaultStyle, noColor = false, outerPad = true): string = + ## Apply a full style object to a string, using the passed style. + ## Does not process Markdown or HTML. + ## + ## Deprecated; use the style API instead. + ## + ## Returns a string. + ## + ## Note that you should never pass strings with control codes to + ## this API. It will not get considered in the state machine / + ## processing done. Not only should you avoid manually adding + ## control codes, this also means you should never feed the output + ## of this API back into the API. + return pre(s).render(width, showLinks, style, ensureNl, noColor, outerPad) proc stylize*(s: string, tag: string, width = -1, showLinks = false, - ensureNl = true, style = defaultStyle): string = + ensureNl = true, style = defaultStyle, noColor = false, + outerPad = true): string = + ## Apply a full style object to a string, specifying a `tag` to + ## use (an html tag) for the style. Does not process Markdown or + ## HTML. + ## + ## Deprecated; use the style API instead. + ## + ## If passed, `style` should be a style object that will be the + ## starting style (after layering it on top of the default style). + ## + ## Any stored style associated with the `tag` parameter will get + ## applied AFTER the style object. + ## + ## Returns a string. + ## + ## Note that you should never pass strings with control codes to + ## this API. It will not get considered in the state machine / + ## processing done. Not only should you avoid manually adding + ## control codes, this also means you should never feed the output + ## of this API back into the API. var r: Rope if tag != "": @@ -147,24 +207,110 @@ proc stylize*(s: string, tag: string, width = -1, showLinks = false, else: r = Rope(kind: RopeAtom, text: s.toRunes()) - return r.preRender(width, showLinks, style). - preRenderBoxToAnsiString(ensureNl) + return r.render(width, showLinks, style, ensureNl, noColor, outerPad) proc withColor*(s: string, fg: string, bg = ""): string = + ## Deprecated. + ## + ## The style API allows you to apply color, chaining the results, + ## but has more extensive options than color. + ## + ## To replace both the fg color and bg color, do: + ## s.fgColor("red").bgColor("white") + ## + ## Or, clear the colors with defaultFg() and defaultBg() + ## + ## Note that you should never pass strings with control codes to + ## this API. It will not get considered in the state machine / + ## processing done. Not only should you avoid manually adding + ## control codes, this also means you should never feed the output + ## of this API back into the API. + + if fg == "" and bg == "": result = s else: result = s.stylize(ensureNl = false, style = newStyle(fgColor = fg, bgColor = bg)) - result = result.strip() +proc `$`*(r: Rope, width = 0, ensureNl = false, showLinks = false, + style = defaultStyle, noColor = false, outerPad = true): string = + ## Default rope-to-string output function. + return r.render(width, showLinks, style, ensureNl, noColor, outerPad) + +proc setvbuf(f: File, buf: pointer, t: cint, s: cint): cint {. importc, + header: "" .} -proc print*(s: string, file = stdout, md = true, width = -1, ensureNl = true, - showLinks = false, style = defaultStyle) = +proc unbufferIo*() = + once: + discard setvbuf(stdout, nil, cint(2), 0) + discard setvbuf(stderr, nil, cint(2), 0) + discard setvbuf(stdin, nil, cint(2), 0) + +proc print*(s: string, file = stdout, forceMd = false, width = 0, + ensureNl = true, showLinks = false, style = defaultStyle, + noAutoDetect = false, noColor = false, outerPad = true) = + unbufferIo() + ## Much like `echo()`, but more capable in terms of the processing + ## you can do. + ## + ## Particularly, `print()` can accept Markdown or HTML, and render + ## it for the terminal to the current terminal width. It can also + ## apply foreground/background colors, and other styling. + ## + ## If a string to print starts with '#', it's assumed to be Markdown + ## (HTML if it starts with '<'). To always skip conversion, then set + ## `noAutoDetect = true`. + ## + ## If you know your string might contain Markdown or HTML, but might + ## not start with a special character, you can instead set + ## `forceMd = true`. + ## + ## Generally, the terminal width is automatically queried at the + ## time you call `print()`, but the `width` parameter allows you + ## to render to a particular width. + ## + ## When true, `showLinks` currently will render both the URL and the + ## text in an html element, using a markdown-like syntax. + ## + ## The `style` parameter allows you to override elements of the + ## default starting style. However, it does NOT override any style + ## formatting set. To do that, use the style API. + ## + ## Unlike `echo()`, where inputs to print can be comma separated, + ## and are automatically converted to strings, `print()` only + ## accepts strings in the first parameter (there's a variant that + ## supports Ropes, essentially strings already + ## + ## The `noColor` flag will inhibit any ansi codes, despite any + ## global settings allowing color. + ## + ## Do not pass strings with control codes as inputs. + ## + ## Markdown is processed by MD4C (which converts it to HTML), and + ## HTML is processd by gumbo. If you use this feature, but pass + ## text that the underlying processor doesn't accept, the result is + ## undefined. Assume you won't get what you want, anyway! + + if s[0] == '\e': + # If it starts with an Ansi code, fall back to echo, as it was + # probably generated with one of the above functions. + # + # But we avoid a linear scan of the entire string. + file.write(s) + return var toWrite: string - if md: - toWrite = s.stylizeMd(width, showLinks, ensureNl, style) - else: - toWrite = s.stylizeHtml(width, showLinks, ensureNl, style) + if forceMd or ((not noAutoDetect) and len(s) > 1 and s[0] == '#'): + toWrite = s.stylizeMd(width, showLinks, ensureNl, style, noColor, outerPad) + elif len(s) >= 1 and s[0] == '<': + toWrite = s.stylizeHtml(width, showLinks, ensureNl, style, noColor, outerPad) + else: + toWrite = pre(s).render(width, showLinks, style, ensureNl, noColor, outerPad) file.write(toWrite) + +proc print*(r: Rope, file = stdout, width = -1, ensureNl = true, + showLinks = false, style = defaultStyle, noColor = false, + outerPad = true) = + unbufferIo() + file.write(r.render(width, showLinks, style, ensureNl, noColor, outerPad)) diff --git a/nimutils/rope_base.nim b/nimutils/rope_base.nim index b4cfcbc..20cbdc1 100644 --- a/nimutils/rope_base.nim +++ b/nimutils/rope_base.nim @@ -1,7 +1,12 @@ -import unicode, tables, options, unicodeid, unicodedb/properties, misc +import unicode, options, unicodeid, unicodedb/properties, misc, strutils + + const defaultTextWidth* {.intdefine.} = 80 bareMinimumColWidth* {.intdefine.} = 2 + StyleColorPop* = 0xfffffffd'u32 + StyleColor* = 0xfffffffd'u32 + StyleNoColor* = 0xfffffffe'u32 StylePop* = 0xffffffff'u32 type @@ -41,7 +46,6 @@ type # For centering, if spaces do not divide evenly, we add the # single extra space to the right. - BorderOpts* = enum BorderNone = 0, BorderTop = 1, @@ -54,6 +58,10 @@ type BorderAll = 8 FmtStyle* = ref object # For terminal formatting. + ## Style objects are all expressed as deltas from a currently + ## active style. When combining styles (with `mergeStyle()`), + ## anything that is a `some()` object will take priority, + ## replacing any set value. textColor*: Option[string] bgColor*: Option[string] overflow*: Option[OverflowPreference] @@ -69,8 +77,6 @@ type italic*: Option[bool] underlineStyle*: Option[UnderlineStyle] bulletChar*: Option[Rune] - minTableColWidth*: Option[int] # Currently not used. - maxTableColWidth*: Option[int] # Currently not used. useTopBorder*: Option[bool] useBottomBorder*: Option[bool] useLeftBorder*: Option[bool] @@ -81,6 +87,12 @@ type alignStyle*: Option[AlignStyle] BoxStyle* = ref object + ## This data structure specifies what Unicode characters to use + ## for different styles of box. Generally, you should not need to + ## do anything custom, as we provide all the standard options in + ## the unicode set. Some of them (particularly the dashed ones) + ## are not universally provided, so should be avoided if you're + ## not providing some sort of accomidation. horizontal*: Rune vertical*: Rune upperLeft*: Rune @@ -95,8 +107,7 @@ type RopeKind* = enum RopeAtom, RopeBreak, RopeList, RopeTable, RopeTableRow, RopeTableRows, - RopeFgColor, RopeBgColor, RopeLink, RopeTaggedContainer, - RopeAlignedContainer + RopeFgColor, RopeBgColor, RopeLink, RopeTaggedContainer BreakKind* = enum # For us, a single new line translates to a soft line break that @@ -106,16 +117,19 @@ type BrSoftLine, BrHardLine, BrParagraph, BrPage ColInfo* = object + ## Currently, we only support percents, so this object is more + ## a placeholder than anything. span*: int widthPct*: int Rope* = ref object - next*: Rope - cycle*: bool - tag*: string - id*: string - class*: string - width*: int # Requested width in columns for a container + ## The core rope object. Generally should only access via API. + next*: Rope + cycle*: bool + noTextExtract*: bool + id*: string + tag*: string + class*: string case kind*: RopeKind of RopeAtom: @@ -129,8 +143,9 @@ type toHighlight*: Rope of RopeList: items*: seq[Rope] - of RopeTaggedContainer, RopeAlignedContainer: + of RopeTaggedContainer: contained*: Rope + width*: int # Requested width in columns for a container. of RopeTable: colInfo*: seq[ColInfo] thead*: Rope # RopeTableRows @@ -148,6 +163,102 @@ type width*: int # Advisory. softBreak*: bool +proc buildWalk(r: Rope, results: var seq[Rope]) = + if r == nil: + return + + results.add(r) + + case r.kind + of RopeBreak: + r.guts.buildWalk(results) + of RopeLink: + r.toHighlight.buildWalk(results) + of RopeList: + for item in r.items: + item.buildWalk(results) + of RopeTaggedContainer: + r.contained.buildWalk(results) + of RopeTable: + r.thead.buildWalk(results) + r.tbody.buildWalk(results) + r.tfoot.buildWalk(results) + r.caption.buildWalk(results) + of RopeTableRow, RopeTableRows: + for item in r.cells: + item.buildWalk(results) + of RopeFgColor, RopeBgColor: + r.toColor.buildWalk(results) + else: + discard + + r.next.buildWalk(results) + +proc ropeWalk*(r: Rope): seq[Rope] = + r.buildWalk(result) + +proc search*(r: Rope, + tag = openarray[string]([]), + class = openarray[string]([]), + id = openarray[string]([]), + text = openarray[string]([]), + first = false): seq[Rope] = + + for item in r.ropeWalk(): + if item.tag in tag or item.class in class or item.id in id: + result.add(item) + if first: + return + elif text.len() != 0 and item.kind == RopeAtom: + for s in text: + if s in $(item.text): + result.add(item) + if first: + return + break + +proc search*(r: Rope, tag = "", class = "", id = "", text = "", + first = false): seq[Rope] = + var tags, classes, ids, texts: seq[string] + + if tag != "": + tags.add(tag) + if class != "": + classes.add(tag) + if id != "": + ids.add(ids) + if text != "": + texts.add(text) + + return r.search(tags, classes, ids, texts) + +proc debugWalk*(r: Rope): string = + var debugId = 0 + if r != nil: + let items = r.ropeWalk() + for item in items: + if item.id == "": + item.id = $(debugId) + debugId += 1 + + for item in r.ropeWalk(): + var one = "id: " & item.id + + if item.kind == RopeAtom: + one &= "; text: " & $(item.text) + else: + one &= "; tag: " + if item.tag == "": + one &= "; " + else: + one &= item.tag + if item.class != "": + one &= "class: " & item.class + + if item.next != nil: + one &= " NEXT = " & item.next.id + result &= one & "\n" + let BoxStylePlain* = BoxStyle(horizontal: Rune(0x2500), vertical: Rune(0x2502), @@ -226,12 +337,40 @@ let bottomT: Rune(0x253b), leftT: Rune(0x2523), rightT: Rune(0x252b)) - + BoxStyleAsterisk* = BoxStyle(horizontal: Rune('*'), + vertical: Rune('*'), + upperLeft: Rune('*'), + upperRight: Rune('*'), + lowerLeft: Rune('*'), + lowerRight: Rune('*'), + cross: Rune('*'), + topT: Rune('*'), + bottomT: Rune('*'), + leftT: Rune('*'), + rightT: Rune('*')) + BoxStyleAscii* = BoxStyle(horizontal: Rune('-'), + vertical: Rune('|'), + upperLeft: Rune('/'), + upperRight: Rune('\\'), + lowerLeft: Rune('\\'), + lowerRight: Rune('/'), + cross: Rune('+'), + topT: Rune('-'), + bottomT: Rune('-'), + leftT: Rune('|'), + rightT: Rune('|')) proc copyStyle*(inStyle: FmtStyle): FmtStyle = + ## Produces a full copy of a style. This is primarily used + ## during the rendering process, but can be used to start + ## with a known style to create another style, without + ## modifying the original style directly. result = FmtStyle(textColor: inStyle.textColor, bgColor: inStyle.bgColor, overflow: inStyle.overFlow, + hang: inStyle.hang, + lpad: instyle.lpad, + rpad: instyle.rpad, tmargin: inStyle.tmargin, bmargin: inStyle.bmargin, casing: inStyle.casing, @@ -241,8 +380,6 @@ proc copyStyle*(inStyle: FmtStyle): FmtStyle = italic: inStyle.italic, underlineStyle: inStyle.underlineStyle, bulletChar: inStyle.bulletChar, - minTableColWidth: inStyle.minTableColWidth, - maxTableColWidth: inStyle.maxTableColWidth, useTopBorder: inStyle.useTopBorder, useBottomBorder: inStyle.useBottomBorder, useLeftBorder: inStyle.useLeftBorder, @@ -256,7 +393,7 @@ proc copyStyle*(inStyle: FmtStyle): FmtStyle = let DefaultBoxStyle* = BoxStyleDouble proc `$`*(plane: TextPlane): string = - # This is more intended for rebugging. + ## Produce a debug representation of a TextPlane object. for line in plane.lines: for ch in line: if ch <= 0x10ffff: @@ -265,18 +402,6 @@ proc `$`*(plane: TextPlane): string = result &= "<<" & $(ch) & ">>" result.add('\n') -proc mergeTextPlanes*(dst: var TextPlane, append: TextPlane) = - if len(dst.lines) == 0: - dst.lines = append.lines - elif len(append.lines) != 0: - dst.lines[^1].add(append.lines[0]) - dst.lines &= append.lines[1 .. ^1] - -proc mergeTextPlanes*(planes: seq[TextPlane]): TextPlane = - result = TextPlane() - for plane in planes: - result.mergeTextPlanes(plane) - proc getBreakOpps(s: seq[uint32]): seq[int] = # Should eventually upgrade this to full Annex 14 at some point. # This is just basic acceptability. If the algorithm finds no @@ -345,6 +470,12 @@ proc getBreakOpps(s: seq[uint32]): seq[int] = result = result[0 ..< ^1] proc stripSpacesButNotFormatters*(input: seq[uint32]): seq[uint32] = + ## Given a sequence of 32-bit integers that contains a combination of + ## runes and formatting markers, strips white space off both sides, + ## but without removing any formatting markers. + ## + ## While subsequent format markers *can* be elided, we haven't + ## bothered yet. for i, ch in input: if ch > 0x10ffff: result.add(ch) @@ -353,6 +484,8 @@ proc stripSpacesButNotFormatters*(input: seq[uint32]): seq[uint32] = return proc stripSpacesButNotFormattersFromEnd*(input: seq[uint32]): seq[uint32] = + ## Same as `stripSpacesButNotFormatters`, but does only removes + ## white space from the end. var n = len(input) while n > 0: n -= 1 @@ -446,6 +579,10 @@ proc ensureFormattingIsPerLine(plane: var TextPlane) = plane.lines[i] &= @[StylePop] proc wrapToWidth*(plane: var TextPlane, style: FmtStyle, w: int) = + ## Wraps the text that's already laid out in a TextPlane object, + ## using the given overflow strategy. Generally, this should + ## probably only be used internally; see `truncateToWidth()` for + ## something that will operate on UTF-32 (arrays of codepoints). # First, we're going to do a basic wrap, without regard to style # indicators. But then we want each line to have the correct stack # state, so we'll go back through and figure out when we need to add diff --git a/nimutils/rope_construct.nim b/nimutils/rope_construct.nim index e6020f7..647c53f 100644 --- a/nimutils/rope_construct.nim +++ b/nimutils/rope_construct.nim @@ -1,12 +1,25 @@ ## :Author: John Viega (john@crashoverride.com) ## :Copyright: 2023, Crash Override, Inc. -import unicode, markdown, htmlparse, tables, parseutils, colortable, rope_base +import unicode, markdown, htmlparse, tables, parseutils, colortable, rope_base, + macros from strutils import startswith, replace - -proc rawStrToRope*(s: string, pre: bool): Rope = +proc textRope*(s: string, pre = false): Rope = + ## Converts a plain string to a rope object. By default, white + ## space is treated as if you stuck the string in an HTML document, + ## which is to mean, line spacing is mostly ignored; spacing between + ## elements is handled by the relationship between adjacent items, + ## like it would be in HTML. + ## + ## To skip this processing, specify `pre = true`, which acts like + ## an HTML
 block.
+  ##
+  ## In both modes, we always replace tabs with four spaces, as
+  ## behavior based on tab-stops isn't supported (much better to use
+  ## table elements and skip tabs all-together). If you don't like it,
+  ## process it before sending it here.
   var
     curStr = ""
     lines: seq[string]
@@ -25,7 +38,7 @@ proc rawStrToRope*(s: string, pre: bool): Rope =
 
     for i, c in s:
       if c == '\t':
-        curStr.add(c)
+        curStr.add("    ")
       elif c == '\n':
         if skipNewline:
           skipNewLine = false
@@ -57,8 +70,10 @@ proc rawStrToRope*(s: string, pre: bool): Rope =
       brk.next  = cur
 
 proc refCopy*(dst: var Rope, src: Rope) =
-  dst.kind = src.kind
-  dst.tag  = src.tag
+  ## Allows you to populate a rope (passed in the first parameter)
+  ## by copying the rope in the second parameter. This copies
+  ## recursively if needed.
+  dst = Rope(kind: src.kind, tag: src.tag)
 
   case src.kind
   of RopeAtom:
@@ -86,7 +101,7 @@ proc refCopy*(dst: var Rope, src: Rope) =
       l.add(sub)
     dst.items = l
 
-  of RopeTaggedContainer, RopeAlignedContainer:
+  of RopeTaggedContainer:
     var sub: Rope = Rope()
     refCopy(sub, src.contained)
     dst.contained = sub
@@ -123,6 +138,12 @@ proc refCopy*(dst: var Rope, src: Rope) =
     dst.next = f
 
 proc `&`*(r1: Rope, r2: Rope): Rope =
+  ## Returns a concatenation of two rope objects, *copying* the
+  ## elements in the rope. This is really only necessary if you might
+  ## end up with cycles in your ropes, or might mutate properties of
+  ## nodes.
+  ##
+  ## Typically, `+` is probably a better bet.
   var
     dupe1: Rope = Rope()
     dupe2: Rope = Rope()
@@ -148,6 +169,12 @@ proc `&`*(r1: Rope, r2: Rope): Rope =
   return dupe1
 
 proc `+`*(r1: Rope, r2: Rope): Rope =
+  ## Returns the concatenation of two ropes, but WITHOUT copying them.
+  ## Unless the first rope is nil, this will return the actual
+  ## left-hand object, so is identical to +=; use & or copy your
+  ## first rope before you use `+` if you want different semantics.
+  ##
+  ## We did it this way, because copying is rarely the right thing.
   if r1 == nil:
     return r2
   if r2 == nil:
@@ -181,6 +208,13 @@ proc `+`*(r1: Rope, r2: Rope): Rope =
 
   return r1
 
+proc `+=`*(r1: var Rope, r2: Rope) =
+  ## Same as `+`
+  if r1 == nil:
+    r1 = r2
+    return
+  r1 = r1 + r2
+
 proc htmlTreeToRope(n: HtmlNode, pre: var seq[bool]): Rope
 
 proc doDescend(n: HtmlNode, pre: var seq[bool]): Rope =
@@ -210,6 +244,10 @@ proc extractColumnInfo(n: HtmlNode): seq[ColInfo] =
 
     result.add(ColInfo(span: span, widthPct: pct))
 
+proc noTextExtract(r: Rope): Rope =
+  r.noTextExtract = true
+  result          = r
+
 proc htmlTreeToRope(n: HtmlNode, pre: var seq[bool]): Rope =
   case n.kind
   of HtmlDocument:
@@ -234,13 +272,19 @@ proc htmlTreeToRope(n: HtmlNode, pre: var seq[bool]): Rope =
           continue
         result.items.add(item.htmlTreeToRope(pre))
     of "right":
-      result = Rope(kind: RopeAlignedContainer, tag: "ralign",
+      result = Rope(kind: RopeTaggedContainer, tag: "right",
                     contained: n.descend())
     of "center":
-      result = Rope(kind: RopeAlignedContainer, tag: "calign",
+      result = Rope(kind: RopeTaggedContainer, tag: "center",
                     contained: n.descend())
     of "left":
-      result = Rope(kind: RopeAlignedContainer, tag: "lalign",
+      result = Rope(kind: RopeTaggedContainer, tag: "left",
+                    contained: n.descend())
+    of "justify":
+      result = Rope(kind: RopeTaggedContainer, tag: "justify",
+                    contained: n.descend())
+    of "flush":
+      result = Rope(kind: RopeTaggedContainer, tag: "flush",
                     contained: n.descend())
     of "thead", "tbody", "tfoot":
       result = Rope(kind:  RopeTableRows, tag: n.contents)
@@ -282,9 +326,10 @@ proc htmlTreeToRope(n: HtmlNode, pre: var seq[bool]): Rope =
           result.cells.add(asRope)
         else: # whitespace colgroup; currently not handling.
           discard
-    of "h1", "h2", "h3", "h4", "h5", "h6", "li", "blockquote", "div",
-       "code", "ins", "del", "kbd", "mark", "q", "s", "small",
+    of "h1", "h2", "h3", "h4", "h5", "h6", "li", "blockquote", "div", "basic",
+       "code", "ins", "del", "kbd", "mark", "p", "q", "s", "small", "td", "th",
        "sub", "sup", "title", "em", "i", "b", "strong", "u", "caption",
+       "text", "plain",
        "var", "italic", "strikethrough", "strikethru", "underline", "bold":
       # Since we know about this list, short-circuit the color checking code,
       # even though if no color matches, the same thing happens as happens
@@ -296,9 +341,6 @@ proc htmlTreeToRope(n: HtmlNode, pre: var seq[bool]): Rope =
       result = Rope(kind: RopeTaggedContainer, tag: n.contents,
                     contained: n.descend())
       discard pre.pop()
-    of "td", "th":
-      result = Rope(kind: RopeTaggedContainer, tag: n.contents,
-                         contained: n.descend())
     else:
       let colorTable = getColorTable()
       let below      = n.descend()
@@ -332,7 +374,7 @@ proc htmlTreeToRope(n: HtmlNode, pre: var seq[bool]): Rope =
         discard parseInt(n.attrs["width"], width)
         result.width = width
   of HtmlText, HtmlCData:
-    result = n.contents.rawStrToRope(pre[^1])
+    result = n.contents.textRope(pre[^1])
   else:
     discard
 
@@ -342,6 +384,18 @@ proc htmlTreeToRope(n: HtmlNode): Rope =
   n.htmlTreeToRope(pre)
 
 proc htmlStringToRope*(s: string, markdown = true): Rope =
+  ## Convert text that is either in HTML or in Markdown into a Rope
+  ## object. If `markdown = false` it will only do HTML conversion.
+  ##
+  ## Markdown conversion works by using MD4C to convert markdown to an
+  ## HTML DOM, and then uses gumbo to produce a tree, which we then
+  ## convert to a Rope (which is itself a tree).
+  ##
+  ## If your input is not well-formed, what you get is
+  ## undefined. Basically, we seem to always get trees of some sort
+  ## from the underlying library, but it may not map to what you want.
+
+
   let html = if markdown: markdownToHtml(s) else: s
   let tree = parseDocument(html).children[1]
 
@@ -353,3 +407,256 @@ proc htmlStringToRope*(s: string, markdown = true): Rope =
     return tree.children[0].children[0].htmlTreeToRope()
   else:
     return tree.htmlTreeToRope()
+
+template html*(s: string): Rope =
+  ## Converts HTML into a Rope object.
+  ##
+  ## If your input is not well-formed, what you get is
+  ## undefined. Basically, we seem to always get trees of some sort
+  ## from the underlying library, but it may not map to what you want.
+  s.htmlStringToRope(markdown = false)
+
+template md*(s: string): Rope =
+  ## An alias for htmlStringToRope, with markdown always true.
+  s.htmlStringToRope(markdown = true)
+macro basicTagGen(ids: static[openarray[string]]): untyped =
+  result = newStmtList()
+
+  for id in ids:
+    let
+      strNode = newLit(id)
+      idNode  = newIdentNode(id)
+      hidNode = newIdentNode("html" & id)
+      decl    = quote do:
+        proc `idNode`*(r: Rope): Rope =
+          ## Apply the style at the point of a rope node.  Sub-nodes
+          ## may override this, but at the time applied, it will
+          ## take priority for the node itself.
+          return Rope(kind: RopeTaggedContainer, tag: `strNode`,
+                      contained: r)
+        proc `idNode`*(s: string): Rope =
+          ## Turn a string into a rope, styled with this tag.
+          return `idNode`(s.textRope(pre = false))
+    result.add(decl)
+
+macro tagGenRename(id: static[string], rename: static[string]): untyped =
+  result = newStmtList()
+
+  let
+    strNode = newLit(id)
+    idNode  = newIdentNode(rename)
+    hidNode = newIdentNode("html" & id)
+    decl    = quote do:
+      proc `idNode`*(r: Rope): Rope =
+        ## Apply the style at the point of a rope node.  Sub-nodes
+        ## may override this, but at the time applied, it will
+        ## take priority for the node itself.
+        return Rope(kind: RopeTaggedContainer, tag: `strNode`,
+                    contained: r)
+      proc `idNode`*(s: string): Rope =
+        ## Turn a string into a rope, styled with this tag.
+        return `idNode`(s.textRope(pre = false))
+  result.add(decl)
+
+macro hidTagGen(ids: static[openarray[string]]): untyped =
+  result = newStmtList()
+
+  for id in ids:
+    let
+      strNode = newLit(id)
+      hidNode = newIdentNode("html" & id)
+      decl    = quote do:
+        proc `hidNode`*(s: string): string =
+          ## Encode the given string in this HTML tag. This does not
+          ## make any attempt to escape contents, and thus should
+          ## generally should only be used as a last resort; prefer
+          ## the interface that returns Rope objects instead (which
+          ## would not have issues with text escaping).
+          return "<" & `strNode` & ">" & s & ""
+
+    result.add(decl)
+
+macro trSetGen(ids: static[openarray[string]]): untyped =
+  result = newStmtList()
+
+  for id in ids:
+    let
+      strNode = newLit(id)
+      idNode  = newIdentNode(id)
+      decl    = quote do:
+        proc `idNode`*(l: seq[Rope]): Rope =
+          ## Converge a set of tr() objects into the proper
+          ## structure expected by table() (which takes only
+          ## one Rope object for this).
+          return Rope(kind: RopeTableRows, tag: `strNode`,
+                      cells: l)
+
+    result.add(decl)
+
+proc tr*(l: seq[Rope]): Rope =
+  ## Converge the passed td() / th() sells into a single row object.
+  ## Pass this to thead(), tbody() or tfoot() only.
+  return Rope(kind: RopeTableRow, tag: "tr", cells: l)
+
+basicTagGen(["h1", "h2", "h3", "h4", "h5", "h6", "li", "blockquote", "div",
+             "container", "code", "ins", "del", "kbd", "mark", "small", "sub",
+             "sup", "width", "title", "em", "strong", "caption", "td", "th",
+             "text", "plain", "deffmt"])
+
+tagGenRename("p",   "paragraph")
+tagGenRename("q",   "quote")
+tagGenRename("u",   "unstructured")
+tagGenRename("var", "variable")
+
+trSetGen(["thead", "tbody", "tfoot"])
+
+hidTagGen(["a", "abbr", "address", "article", "aside", "b", "base", "bdi",
+           "bdo", "blockquote", "br", "caption", "center", "cite", "code",
+           "col", "colgroup", "data", "datalist", "dd", "details", "dfn",
+           "dialog", "dl", "dt", "em", "embed", "fieldset", "figcaption",
+           "figure", "footer", "form", "h1", "h2", "h3", "h4", "h5", "h6",
+           "header", "hr", "i", "ins", "kbd", "label", "legend", "li",
+           "link", "main", "mark", "menu", "meta", "meter", "nav", "ol",
+           "optgroup", "output", "p", "param", "pre", "progress", "q", "s",
+           "samp", "search", "section", "select", "span", "strong", "style",
+           "sub", "summary", "sup", "table", "tbody", "td", "tfoot",
+           "th", "thead", "title", "tr", "u", "ul", "text", "plain"])
+
+proc pre*(r: Rope): Rope =
+  ## This is generally a no-op on a rope object; pre-formatting
+  ## happens when text is initially imported. If you add special
+  ## styling for 'pre' though, it will get applied.
+  return Rope(kind: RopeTaggedContainer, tag: "pre", contained: r)
+
+proc pre*(s: string): Rope =
+  ## Creates a new rope from text, without removing spacing.  This is
+  ## essentially an abbreviation for s.textRope(pre = true), though
+  ## it does also add a `pre` container node, which will pick up
+  ## any styling you choose to apply to that tag.
+  return pre(s.textRope(pre = true))
+
+proc ol*(l: seq[Rope]): Rope =
+  ## Taking a list of li() Ropes, returns a Rope for an ordered (i.e.,
+  ## numbered) list. Currently, there is no way to change the
+  ## numbering style, or to continue numbering from previous lists.
+  return Rope(kind: RopeList, tag: "ol", items: l)
+
+proc ol*(l: seq[string]): Rope =
+  ## Taking a list of strings, it creates a rope for an ordered list.
+  ## The list items are automatically wrapped in li() nodes, but are
+  ## otherwise unprocessed.
+  var listItems: seq[Rope]
+  for item in l:
+    listItems.add(li(item))
+  return ol(listItems)
+
+proc ul*(l: seq[Rope]): Rope =
+  ## Taking a list of li() Ropes, returns a Rope for an unordered
+  ## (i.e., bulleted) list.
+  return Rope(kind: RopeList, tag: "ul", items: l)
+
+proc ensureNewline*(r: Rope): Rope {.discardable.} =
+  ## Used to wrap terminal output when ensureNl is true, but the
+  ## content is not enclosed in a basic block. This is done using
+  ## a special 'basic' tag.
+  return Rope(kind: RopeTaggedContainer, tag: "basic", contained: r)
+
+proc ul*(l: seq[string]): Rope =
+  ## Taking a list of strings, it creates a rope for a bulleted list.
+  ## The list items are automatically wrapped in li() nodes, but are
+  ## otherwise unprocessed.
+  var listItems: seq[Rope]
+  for item in l:
+    listItems.add(li(item))
+  return ul(listItems)
+
+proc setWidth*(r: Rope, i: int): Rope =
+  ## Returns a rope that constrains the passed Rope to be formatted
+  ## within a particular width, as long as the context in which the
+  ## rope's being evaluated has at least that much width available.
+  return Rope(kind: RopeTaggedContainer, tag: "width", contained: r,
+              width: i, noTextExtract: true)
+
+proc setWidth*(s: string, i: int): Rope =
+  ## Returns a rope that constrains the passed string to be formatted
+  ## within a particular width, as long as the context in which the
+  ## rope's being evaluated has at least that much width available.
+  result = noTextExtract(pre(s)).setWidth(i)
+
+proc table*(tbody: Rope, thead: Rope = nil, tfoot: Rope = nil,
+            caption: Rope = nil, columnInfo: seq[ColInfo] = @[]): Rope =
+  ## Generates a Rope that outputs a table. The content parameters
+  ## must be created by tbody(), thead() or tfoot(), or else you will
+  ## get an error from the internals (we do not explicitly check for
+  ## this mistake right now).
+  ##
+  ## For the caption, you *should* provide a caption() object if you
+  ## want it to be styled appropriately, but this one should not error
+  ## if you don't.
+  ##
+  ## The `columnInfo` field can be set by calling `colPcts`
+  ## (currently, we only support percentage based widths, and do not
+  ## support column spans or row spans).
+  ##
+  ## Note that, for various reasons, table style often will not get
+  ## applied the way you might expect. To counter that, We wrapped
+  ## tables in a generic `container()` node.
+  result = container(Rope(kind: RopeTable, tag: "table", tbody: tbody,
+                          thead: thead, tfoot: tfoot, caption: caption,
+                          colInfo: columnInfo))
+
+proc colPcts*(pcts: openarray[int]): seq[ColInfo] =
+  ## This takes a list of column percentages and returns what you need
+  ## to pass to `table()`.
+  ##
+  ## You can alternately call colPcts() on an existing rope object where
+  ## no pcts had been applied before.
+  ##
+  ## Column widths are determined dynamically when rendering, based on
+  ## the available size that we're asked to render into. The given
+  ## percentage is used to calculate how much space to use for a
+  ## column.
+  ##
+  ## Percents do not need to add up to 100.
+  ##
+  ## If you specify a column's with to be 0, this is taken to be the
+  ## 'default' width, which we calculate by dividing any space not
+  ## allocated to other columns evenly, and giving the same amount to
+  ## each column (rounding down if there's no even division).
+  ##
+  ## However, if there is no room for a default column, we give it a
+  ## minimum size of two characters. There is currently no facility
+  ## for hiding columns. However, any columns that extend to the right
+  ## of the available width will end up truncated.
+  ##
+  ## Specified percents can also go above 100, but you will see
+  ## truncation there as well.
+  for item in pcts:
+    result.add(ColInfo(widthPct: item, span: 1))
+
+proc colors*(r: Rope, removeNested = true): Rope =
+  ## Unless no-color is off, use coloring for this item, when
+  ## available. The renderer determines what this means.
+  ##
+  ## This is specifically meant for the terminal, where this gets
+  ## interpreted as "don't show any ansi codes at all".
+  ##
+  ## This does NOT suspend other style processing.
+  result = Rope(kind: RopeTaggedContainer, tag: "colors", contained: r)
+  if removeNested:
+    for item in r.search("nocolors"):
+      item.tag = "colors"
+
+proc nocolors*(r: Rope, removeNested = true): Rope =
+  ## Explicitly turns off any coloring for this item. However, this is
+  ## loosely interpreted; on a terminal it will also turn off any other
+  ## ansi codes being used.
+  ##
+  ## This is specifically meant for the terminal, where this gets
+  ## interpreted as "don't show any ansi codes at all".
+  ##
+  ## This does NOT suspend other style processing.
+  result = Rope(kind: RopeTaggedContainer, tag: "nocolors", contained: r)
+  if removeNested:
+    for item in r.search("colors"):
+      item.tag = "nocolors"
diff --git a/nimutils/rope_prerender.nim b/nimutils/rope_prerender.nim
index 4d1dd02..f255077 100644
--- a/nimutils/rope_prerender.nim
+++ b/nimutils/rope_prerender.nim
@@ -26,8 +26,8 @@
 # single-threaded assumption until I bring in my lock-free hash
 # tables.
 
-import tables, options, unicodedb/properties, std/terminal, rope_base,
-       rope_styles, unicodeid, unicode, misc
+import tables, options, std/terminal, rope_base, rope_styles, unicodeid,
+       unicode, misc
 
 type
   RenderBoxKind* = enum RbText, RbBoxes
@@ -39,15 +39,16 @@ type
     width*:      int
 
   FmtState = object
-    totalWidth:   int
-    showLinkTarg: bool
-    curStyle:     FmtStyle
-    styleStack:   seq[uint32]
-    colStack:     seq[seq[int]]
-    nextRope:     Rope
-    savedRopes:   seq[Rope]
-    processed:    seq[Rope] # For text items b/c I have a bug :/
-    tableEven:    seq[bool]
+    totalWidth:      int
+    showLinkTarg:    bool
+    curStyle:        FmtStyle
+    styleStack:      seq[uint32]
+    colStack:        seq[seq[int]]
+    colorStack:      seq[bool]
+    tableEven:       seq[bool]
+    curPlane:        TextPlane
+    curContainer:    Rope
+    curTableSep:     Option[RenderBox]
 
 proc `$`*(box: RenderBox): string =
     result &= $(box.contents)
@@ -56,50 +57,69 @@ proc `$`*(box: RenderBox): string =
 template styleRunes(state: FmtState, runes: seq[uint32]): seq[uint32] =
   @[state.curStyle.getStyleId()] & runes & @[StylePop]
 
-proc applyCurrentStyleToPlane*(state: FmtState, p: TextPlane) =
-  for i in 0 ..< p.lines.len():
-    p.lines[i] = state.styleRunes(p.lines[i])
-
 template pad(state: FmtState, w: int): seq[uint32] =
   state.styleRunes(uint32(Rune(' ')).repeat(w))
 
 proc noBoxRequired*(r: Rope): bool =
+  ## Generally, this call is only meant to be used either internally,
+  ## or by a renderer (the ansi renderer being the only one we
+  ## currently have).
+  ##
   ## Returns true if we have paragraph text that does NOT require any
   ## sort of box... so no alignment, padding, tables, lists, ...
   ##
   ## However, we DO allow break objects, as they don't require boxing,
   ## so it isn't quite non-breaking text.
+  ##
+  ## This has gotten a bit more complicated with the styling
+  ## API. Previously we relied on the tag being in 'breaking
+  ## styles'. However, with the style API, one can easily set
+  ## properties that change whether a box is implied. So, while we
+  ## still check the list of tags that imply a box, we also check the
+  ## boolean `noTextExtract`.
+  ##
+  ## This boolean isn't meant to be definitive; it's only to be added
+  ## to nodes that will short-circuit text extraction, so that box
+  ## properties get applied, and we don't bother to set it when the
+  ## tag already iplies it.
 
   # This will be used to test containers that may contain some basic
   # content, some not.
 
-  var subItem: Rope
-
+  if r == nil:
+    return true
+  if r.tag in breakingStyles or r.noTextExtract:
+    return false
   case r.kind
-  of RopeList, RopeAlignedContainer, RopeTable, RopeTableRow, RopeTableRows:
+  of RopeList, RopeTable, RopeTableRow, RopeTableRows:
     return false
-  of RopeAtom, RopeLink:
-    return true
+  of RopeAtom:
+    result = true
+  of RopeLink:
+    result = r.toHighlight.noBoxRequired()
   of RopeFgColor, RopeBgColor:
-    # If our contained item is basic, we need to check the
-    # subsequent items too.
-    # Assign to subItem and drop down to the loop below.
-    subItem = r.toColor
+    result = r.toColor.noBoxRequired()
   of RopeBreak:
-    return r.guts == Rope(nil)
+    result = r.guts == Rope(nil)
   of RopeTaggedContainer:
-    if r.tag in breakingStyles:
-      return false
-    subItem = r.contained
+    result = r.contained.noBoxRequired()
 
-  while subItem != Rope(nil):
-    if not subItem.noBoxRequired():
-      return false
-    subItem = subItem.next
-
-  return true
+  if result != false:
+    result = r.next.noBoxRequired()
 
 proc unboxedRunelength*(r: Rope): int =
+  ## Returns the approximate display-width of a rope, without
+  ## considering the size of 'box' we're going to try to fit it into.
+  ##
+  ## That is, this call returns how many characters of fixed-sized
+  ## width we think we need to render the given rope.
+  ##
+  ## Note that we cannot ultimately know how a terminal will render a
+  ## given string, especially when it comes to Emoji. Under the hood,
+  ## we do our best, but stick to expected values provided in the
+  ## Unicode standard. But there may occasionally be length
+  ## calculation issues due to local fonts, etc.
+
   if r == Rope(nil):
     return 0
   case r.kind
@@ -119,7 +139,11 @@ proc unboxedRunelength*(r: Rope): int =
   else:
     return 0
 
-proc applyAlignment(state: FmtState, box: RenderBox, w: int) =
+template runelength*(r: Rope): int = r.unboxedRuneLength()
+
+proc applyAlignment(state: FmtState, box: RenderBox) =
+  let w = state.totalWidth
+
   for i in 0 ..< box.contents.lines.len():
     let
       toFill =  w - box.contents.lines[i].u32LineLength()
@@ -149,28 +173,34 @@ proc applyAlignment(state: FmtState, box: RenderBox, w: int) =
     else:
       discard
 
-proc applyLeftRightPadding(state: FmtState, box: RenderBox, w: int) =
+proc applyPadding(state: FmtState, box: RenderBox, lpad, rpad: int) =
+  ## When we're applying a container style, the contents are rendered
+  ## to a width calculated after subtracting out the padding.
+  ##
+  ## When this is called, the state object is at the end of applying
+  ## the style, and we're going to go ahead and make sure each line
+  ## is exactly the right width (to the best of our ability due to
+  ## unicode issues), then add the padding on.
   var
-    lpad    = state.curStyle.lpad.getOrElse(0)
-    rpad    = state.curStyle.rpad.getOrElse(0)
     lpadTxt = state.pad(lpad)
     rpadTxt = state.pad(rpad)
-    extra: seq[uint32]
+    w       = state.totalWidth
+    toFill: int
 
   for i in 0 ..< len(box.contents.lines):
-    var toFill = (w - box.contents.lines[i].u32LineLength())
-    extra = state.pad(toFill)
-    box.contents.lines[i] = lpadTxt & box.contents.lines[i] & rpadTxt & extra
-    # There's a bug if this is needed.
-    box.contents.lines[i] = box.contents.lines[i].truncateToWidth(w)
+    toFill = (w - box.contents.lines[i].u32LineLength())
 
-proc alignAndPad(state: FmtState, box: RenderBox) =
-  let
-    lpad    = state.curStyle.lpad.getOrElse(0)
-    rpad    = state.curStyle.rpad.getOrElse(0)
+    if toFill < 0:
+      box.contents.lines[i] = box.contents.lines[i].truncateToWidth(w)
+    elif toFill > 0:
+      box.contents.lines[i] = box.contents.lines[i] & state.pad(toFill)
 
-  state.applyAlignment(box, state.totalWidth - (lpad - rpad))
-  state.applyLeftRightPadding(box, state.totalWidth)
+    assert box.contents.lines[i].u32LineLength() == w
+    box.contents.lines[i] = lpadTxt & box.contents.lines[i] & rpadTxt
+
+proc wrapTextPlane(p: TextPlane, lineStart, lineEnd: seq[uint32]) =
+  for i in 0 ..< p.lines.len():
+    p.lines[i] = lineStart & p.lines[i] & lineEnd
 
 proc collapseColumn(state: FmtState, boxes: seq[RenderBox]): RenderBox =
   ## Combine renderboxes at the same level into one renderbox.  These
@@ -181,31 +211,22 @@ proc collapseColumn(state: FmtState, boxes: seq[RenderBox]): RenderBox =
   var plane: TextPlane = TextPlane()
   let
     style   = state.curStyle
-    lineLen = state.totalWidth - style.lpad.get(0) - style.rpad.get(0)
-    blank   = state.pad(lineLen)
-    tmargin = if boxes.len() != 0: boxes[0].tmargin else: 0
-    bmargin = if boxes.len() != 0: boxes[0].bmargin else: 0
+    blank   = state.pad(state.totalWidth)
 
   for i, box in boxes:
-    if i != 0:
-      for j in 0 ..< box.tmargin:
-        plane.lines.add(blank)
+    for j in 0 ..< box.tmargin:
+      plane.lines.add(blank)
 
     plane.lines &= box.contents.lines
 
-    if i != len(boxes) - 1:
-      for j in 0 ..< box.bmargin:
-        plane.lines.add(blank)
+    for j in 0 ..< box.bmargin:
+      plane.lines.add(blank)
 
-  result = RenderBox(contents: plane, tmargin: tmargin, bmargin: bmargin)
+  result = RenderBox(contents: plane)
 
 proc collapsedBoxToTextPlane(state: FmtState, box: RenderBox): TextPlane =
   result       = box.contents
   result.width = box.width
-  for i in 0 ..< box.tmargin:
-    result.lines = @[state.pad(result.width)] & result.lines
-  for i in 0 ..< box.bmargin:
-    result.lines &= @[state.pad(result.width)]
 
 proc pushTableWidths(state: var FmtState, widths: seq[int]) =
   state.colStack.add(widths)
@@ -225,8 +246,7 @@ proc popStyle(state: var FmtState) =
   else:
     state.curStyle = defaultStyle
 
-proc getNewStartStyle(state: FmtState, r: Rope,
-                      otherTag = ""): Option[FmtStyle] =
+proc getNewStartStyle(state: FmtState, r: Rope): Option[FmtStyle] =
   # First, apply any style object associated with the rope's html tag.
   # Second, if the rope has a class, apply any style object associated w/ that.
   # Third, do the same w/ ID.
@@ -236,10 +256,9 @@ proc getNewStartStyle(state: FmtState, r: Rope,
     styleChange = false
     newStyle    = state.curStyle
 
-  if otherTag != "" and otherTag in styleMap:
-    styleChange = true
-    newStyle = newStyle.mergeStyles(styleMap[otherTag])
-  elif r != nil and r.tag != "" and r.tag in styleMap:
+  if r == nil:
+    return none(FmtStyle)
+  if r.tag != "" and r.tag in styleMap:
     styleChange = true
     newStyle = newStyle.mergeStyles(styleMap[r.tag])
   if r != nil and r.class != "" and r.class in perClassStyles:
@@ -259,76 +278,69 @@ proc getNewStartStyle(state: FmtState, r: Rope,
   if styleChange:
     return some(newStyle)
 
-template boxContent(state: var FmtState, style: FmtStyle, symbol: untyped,
-                    code: untyped) =
-  state.pushStyle(style)
-
-  let
-    lpad     = style.lpad.getOrElse(0)
-    rpad     = style.rpad.getOrElse(0)
-    p        = lpad + rpad
-
-  state.totalWidth -= p
+template addStyleMarkers(code: untyped) =
+  let style = state.getNewStartStyle(r).getOrElse(state.curStyle)
+  state.curPlane.addRunesToExtraction(@[state.curStyle.getStyleId()])
   code
-  state.totalWidth += p
+  state.curPlane.addRunesToExtraction(@[StylePop])
 
-  for item in symbol:
-    if style.tmargin.isSome():
-      item.tmargin = style.tmargin.get()
-    if style.bmargin.isSome():
-      item.bmargin = style.bmargin.get()
-
-  let collapsed = state.collapseColumn(symbol)
-  state.alignAndPad(collapsed)
+template withRopeStyle(code: untyped) =
+  let style = state.getNewStartStyle(r).getOrElse(state.curStyle)
+  state.pushStyle(style)
+  code
   state.popStyle()
 
-  symbol = @[collapsed]
+template withWidth(w: int, code: untyped) =
+  let oldWidth = state.totalWidth
 
-template fmtBox(styleTweak: Option[FmtStyle], code: untyped) =
-  var style: FmtStyle
+  if w < 0:
+    w = 0
+  elif w < oldwidth:
+    state.totalWidth = w
 
-  if styleTweak.isSome():
-    style = state.curStyle.mergeStyles(styleTweak.get())
-  else:
-    style = state.curStyle
-  state.boxContent(style, result, code)
+  code
 
-template taggedBox(tag: string, code: untyped) =
-  let
-    styleOpt   = state.getNewStartStyle(r, tag)
-    style      = styleOpt.getOrElse(state.curStyle)
-    savedWidth = state.totalWidth
-    lpad       = style.lpad.getOrElse(0)
-    rpad       = style.rpad.getOrElse(0)
-    p          = lpad + rpad
+  state.totalWidth = oldWidth
 
-  state.pushStyle(style)
+template flushCurPlane(boxvar: untyped) =
+  if state.curPlane.lines.len() != 0:
+    state.curPlane.wrapToWidth(state.curStyle, state.totalWidth)
+    boxvar.add(RenderBox(contents: state.curPlane))
+    state.curPlane = TextPlane(lines: @[])
 
-  if r.width != 0 and r.width < state.totalWidth:
-    state.totalWidth = r.width - p
-  else:
-    state.totalWidth -= p
+template enterContainer(boxvar, code: untyped) =
+  flushCurPlane(boxvar)
+  let savedContainer = state.curContainer
+  state.curContainer = r
   code
-  state.totalWidth = savedWidth
-  for item in result:
-    if style.tmargin.isSome():
-      item.tmargin = style.tmargin.get()
-    if style.bmargin.isSome():
-      item.bmargin = style.bmargin.get()
-
-  let collapsed = state.collapseColumn(result)
-  state.alignAndPad(collapsed)
-  state.popStyle()
+  state.curContainer = savedContainer
 
-  result = @[collapsed]
+template applyContainerStyle(boxvar: untyped, code: untyped) =
+  var
+    style = state.getNewStartStyle(r).getOrElse(state.curStyle)
+    lpad  = style.lpad.getOrElse(0)
+    rpad  = style.rpad.getOrElse(0)
+    tpad  = style.tmargin.getOrElse(0)
+    bpad  = style.bmargin.getOrElse(0)
+    w     = state.totalWidth - lpad - rpad
+    collapsed: RenderBox
+
+  state.pushStyle(style)
+  withWidth(w):
+    code
+    flushCurPlane(boxvar)
+    collapsed = state.collapseColumn(boxvar)
+    collapsed.tmargin = tpad
+    collapsed.bmargin = bpad
+    state.applyAlignment(collapsed)
+    state.applyPadding(collapsed, lpad, rpad)
 
-template standardBox(code: untyped) =
-  taggedBox("", code)
+  state.popStyle()
+  boxvar = @[collapsed]
 
 proc preRender*(state: var FmtState, r: Rope): seq[RenderBox]
 
 proc preRenderUnorderedList(state: var FmtState, r: Rope): seq[RenderBox] =
-  standardBox:
     let
       bulletChar = state.curStyle.bulletChar.getOrElse(Rune(0x2022))
       bullet     = state.styleRunes(@[uint32(bulletChar)])
@@ -343,7 +355,7 @@ proc preRenderUnorderedList(state: var FmtState, r: Rope): seq[RenderBox] =
       subedWidth = false
 
     for n, item in r.items:
-      var oneItem = state.preRender(item)[0]
+      var oneItem  = state.preRender(item)[0]
 
       for i in 0 ..< oneItem.contents.lines.len():
         if i == 0:
@@ -372,7 +384,6 @@ proc toNumberBullet(state: FmtState, n, maxdigits: int): seq[uint32] =
   result = pad & state.styleRunes(result)
 
 proc preRenderOrderedList(state: var FmtState, r: Rope): seq[RenderBox] =
-  standardBox:
     var
       hangPrefix:  seq[uint32]
       maxDigits  = 0
@@ -434,45 +445,45 @@ proc getGenericBorder(state: var FmtState, colWidths: seq[int],
                       style: FmtStyle, horizontal: Rune,
                       leftBorder: Rune, rightBorder: Rune,
                       sep: Rune): RenderBox =
-  let
-    useLeft  = style.useLeftBorder.getOrElse(false)
-    useRight = style.useRightBorder.getOrElse(false)
-    useSep   = style.useVerticalSeparator.getOrElse(false)
+    let
+      useLeft  = style.useLeftBorder.getOrElse(false)
+      useRight = style.useRightBorder.getOrElse(false)
+      useSep   = style.useVerticalSeparator.getOrElse(false)
 
-  var
-    plane = TextPlane(lines: @[@[]])
+    var
+      plane = TextPlane(lines: @[@[]])
 
-  if useLeft:
-    plane.lines[0] &= @[uint32(leftBorder)]
+    if useLeft:
+      plane.lines[0] &= @[uint32(leftBorder)]
 
-  for i, width in colWidths:
-    for j in 0 ..< width:
-      plane.lines[0].add(uint32(horizontal))
-    if useSep and i != len(colWidths) - 1:
-      plane.lines[0].add(uint32(sep))
+    for i, width in colWidths:
+      for j in 0 ..< width:
+        plane.lines[0].add(uint32(horizontal))
+      if useSep and i != len(colWidths) - 1:
+        plane.lines[0].add(uint32(sep))
 
-  if useRight:
-    plane.lines[0].add(uint32(rightBorder))
-    plane.lines[0] = state.styleRunes(plane.lines[0])
+    if useRight:
+      plane.lines[0].add(uint32(rightBorder))
+      plane.lines[0] = state.styleRunes(plane.lines[0])
 
-  result = RenderBox(contents: plane)
+    result = RenderBox(contents: plane)
 
-template getTopBorder(state: var FmtState, s: BoxStyle): RenderBox =
-  state.getGenericBorder(state.colStack[^1], state.curStyle, s.horizontal,
-                         s.upperLeft, s.upperRight, s.topT)
+proc getTopBorder(state: var FmtState, s: BoxStyle): RenderBox =
+  result = state.getGenericBorder(state.colStack[^1], state.curStyle,
+           s.horizontal, s.upperLeft, s.upperRight, s.topT)
 
-template getHorizontalSep(state: var FmtState, s: BoxStyle): RenderBox =
-  state.getGenericBorder(state.colStack[^1], state.curStyle, s.horizontal,
-                         s.leftT, s.rightT, s.cross)
+proc getHorizontalSep(state: var FmtState, s: BoxStyle): RenderBox =
+  result = state.getGenericBorder(state.colStack[^1], state.curStyle,
+           s.horizontal, s.leftT, s.rightT, s.cross)
 
-template getBottomBorder(state: var FmtState, s: BoxStyle): RenderBox =
-  state.getGenericBorder(state.colStack[^1], state.curStyle, s.horizontal,
-                         s.lowerLeft, s.lowerRight, s.bottomT)
+proc getBottomBorder(state: var FmtState, s: BoxStyle): RenderBox =
+  result = state.getGenericBorder(state.colStack[^1], state.curStyle,
+           s.horizontal, s.lowerLeft, s.lowerRight, s.bottomT)
 
 proc preRenderTable(state: var FmtState, r: Rope): seq[RenderBox] =
-  standardBox:
     var
       colWidths: seq[int]
+      savedSep  = state.curTableSep
       boxStyle  = state.curStyle.boxStyle.getOrElse(DefaultBoxStyle)
 
     if r.colInfo.len() != 0:
@@ -502,40 +513,46 @@ proc preRenderTable(state: var FmtState, r: Rope): seq[RenderBox] =
       for i, width in colWidths:
         if width < 2:
           colWidths[i] = 2
+    else:
+      let rows = r.search("tr", first = true)
+      if len(rows) != 0:
+        let
+          row = rows[0]
+          pct = 100 div len(row.cells)
+        for i in 0 ..< len(row.cells):
+          colWidths.add(pct)
 
     state.pushTableWidths(state.percentToActualColumns(colWidths))
 
-    if r.thead != Rope(nil):
-      result &= state.preRender(r.thead)
-    if r.tbody != Rope(nil):
-      result &= state.preRender(r.tbody)
-    if r.tfoot != Rope(nil):
-      result &= state.preRender(r.tfoot)
-
     var
       topBorder = state.getTopBorder(boxStyle)
-      lowBorder = state.getBottomBorder(boxStyle)
       midBorder = state.getHorizontalSep(boxStyle)
+      lowBorder = state.getBottomBorder(boxStyle)
 
-    if state.curStyle.useHorizontalSeparator.getOrElse(false):
-      var newBoxes: seq[RenderBox]
+    if r.caption != Rope(nil):
+      result = state.preRender(r.caption)
 
-      for i, item in result:
-        newBoxes.add(item)
-        if (i + 1) != len(result):
-          newBoxes.add(midBorder)
+    if state.curStyle.useTopBorder.getOrElse(false):
+      result &= @[topBorder]
 
-      result = newBoxes
+    if state.curStyle.useHorizontalSeparator.getOrElse(false):
+      state.curTableSep = some(midBorder)
+    else:
+      state.curTableSep = none(RenderBox)
 
-    if state.curStyle.useTopBorder.getOrElse(false):
-      result = @[topBorder] & result
+    if r.thead != Rope(nil):
+      result &= state.preRender(r.thead)
+    if r.tbody != Rope(nil):
+      result &= state.preRender(r.tbody)
+    if r.tfoot != Rope(nil):
+      result &= state.preRender(r.tfoot)
+
+    state.curTableSep = savedSep
 
     if state.curStyle.useBottomBorder.getOrElse(false):
-      result.add(lowBorder)
+        result.add(lowBorder)
 
     state.popTableWidths()
-    if r.caption != Rope(nil):
-      result &= state.preRender(r.caption)
 
 proc emptyTableCell(state: var FmtState): seq[RenderBox] =
   var styleId: uint32
@@ -546,7 +563,8 @@ proc emptyTableCell(state: var FmtState): seq[RenderBox] =
   else:
     styleId = state.curStyle.getStyleId()
 
-  let pane = TextPlane(lines: @[@[styleId, StylePop]])
+  let pane = TextPlane(lines: @[@[styleId,
+                                  StylePop]])
 
   return @[ RenderBox(width: state.totalWidth, contents: pane) ]
 
@@ -571,7 +589,7 @@ proc adjacentCellsToRow(state: var FmtState, cells: seq[TextPlane],
 
   # Determine how many text lines high this row is by examining the
   # height of each cell.
-  for col in cells:
+  for i, col in cells:
     let l = col.lines.len()
     if l > rowLines:
       rowLines = l
@@ -590,6 +608,8 @@ proc adjacentCellsToRow(state: var FmtState, cells: seq[TextPlane],
 
   # Now we go left to right, line-by-line and assemble the result.  We
   # do this by mutating boxes[0]'s state; we'll end up returning it.
+  # However, there's one hitch, which is that, if padding got added to
+  # a cell, we did that by appending a newline.
   for n in 0 ..< rowLines:
     if len(cells) == 1:
       cells[0].lines[n] = leftBorder & cells[0].lines[n]
@@ -610,51 +630,35 @@ proc adjacentCellsToRow(state: var FmtState, cells: seq[TextPlane],
 
 proc preRenderRow(state: var FmtState, r: Rope): seq[RenderBox] =
   # This is the meat of the table implementation.
-  # 1) If the table colWidths array is 0, then we need to
-  #    set it based on our # of columns.
-  # 2) Pre-render the individual cells to the required width.
+  #
+  # 1) Pre-render the individual cells to the required width.
   #    This will return a seq[RenderBox]
-  # 3) Flatten into a TextPlane.
-  # 4) Combine the textplanes horizontally, adding vertical borders.
+  # 2) Flatten into a TextPlane.
+  # 3) Combine the textplanes horizontally, adding vertical borders.
   #
   # This will result in our table being one TextPlane in a single
   # RenderBox.
-  var tag = if state.tableEven[^1]: "tr.even" else: "tr.odd"
-
-  taggedBox(tag):
-    var
-      widths = state.colStack[^1]
+  var
+    tag        = if state.tableEven[^1]: "tr.even" else: "tr.odd"
+    widths     = state.colStack[^1]
+    savedWidth = state.totalWidth
+    cellBoxes: seq[RenderBox]
+    rowPlanes: seq[TextPlane]
 
-    # Step 1, make sure col widths are right
-    if widths.len() == 0:
-      state.popTableWidths()
-      let pct = 100 div len(r.cells)
-      for i in 0 ..< len(r.cells):
-        widths.add(pct)
-      widths = state.percentToActualColumns(widths)
-      state.pushTableWidths(widths)
 
-    var
-      cellBoxes: seq[RenderBox]
-      rowPlanes: seq[TextPlane]
-      savedWidth = state.totalWidth
-
-    # This loop does steps 2-3
-    for i, width in widths:
-      # Step 2, pre-render the cell.
-      if i >= len(r.cells):
-        cellBoxes = state.emptyTableCell()
-      else:
-        state.totalWidth = width
-        cellBoxes = state.preRender(r.cells[i])
+  # This loop does steps 1-2
+  for i, width in widths:
+    # Pre-render the cell.
+    if i >= len(r.cells):
+      cellBoxes = state.emptyTableCell()
+    else:
+      state.totalWidth = width
+      cellBoxes = state.preRender(r.cells[i])
 
-      for cell in cellBoxes:
-        cell.tmargin = 0
-        cell.bmargin = 0
-      let boxes = state.collapseColumn(cellBoxes)
-      rowPlanes.add(state.collapsedBoxToTextPlane(boxes))
+    let boxes = state.collapseColumn(cellBoxes)
+    rowPlanes.add(state.collapsedBoxToTextPlane(boxes))
 
-  # Step 4, Combine the cells horizontally into a single RbText
+  # Step 3, Combine the cells horizontally into a single RbText
   # object. This involves adding any vertical borders, and filling
   # in any blank lines if individual cells span multiple lines.
   let resPlane     = state.adjacentCellsToRow(rowPlanes, widths)
@@ -662,10 +666,15 @@ proc preRenderRow(state: var FmtState, r: Rope): seq[RenderBox] =
   state.totalWidth = savedWidth
 
 proc preRenderRows(state: var FmtState, r: Rope): seq[RenderBox] =
-  # Each row returns a single item.
+  # Each row returns a single item. # We need to make sure to combine
+  # rows properly; if there's a horizontal border in the state,
+  # we add it after all but the last row.
   state.tableEven.add(false)
-  for item in r.cells:
+  for i, item in r.cells:
     result &= state.preRender(item)
+    if i != r.cells.len() - 1 and state.curTableSep.isSome():
+      result &= state.curTableSep.get()
+
     state.tableEven.add(not state.tableEven.pop())
   discard state.tableEven.pop()
 
@@ -686,175 +695,128 @@ template addRunesToExtraction(extraction: TextPlane,
                               runes:      seq[uint32]) =
   extraction.addRunesToExtraction(cast[seq[Rune]](runes))
 
-template subextract(subExtractField: untyped) =
-  state.extractText(subExtractField, extract)
+proc preRenderAtom(state: var FmtState, r: Rope) =
+  withRopeStyle:
+    addStyleMarkers:
+      state.curPlane.addRunesToExtraction(r.text)
 
-template addStyledText(code: untyped) =
-  extract.addRunesToExtraction(@[state.curStyle.getStyleId()])
-  code
-  extract.addRunesToExtraction(@[StylePop])
-
-proc extractText(state: var FmtState, r: Rope, extract: TextPlane) =
-  var cur: Rope = r
-  while cur != nil:
-    if r in state.processed:
-      return
-    state.processed.add(r)
-    case cur.kind
-    of RopeAtom:
-      let styleOpt = state.getNewStartStyle(cur)
-      state.pushStyle(styleOpt.get(state.curStyle))
-      addStyledText(extract.addRunesToExtraction(cur.text))
-      state.popStyle()
-    of RopeLink:
-      let urlRunes = if state.showLinkTarg:
-                       @[Rune('(')] & cur.url.toRunes() & @[Rune(')')]
-                     else:
-                       @[]
-
-      subextract(cur.toHighlight)
-      addStyledText(extract.addRunestoExtraction(urlRunes))
-    else:
-      if not cur.noBoxRequired():
-        state.savedRopes.add(state.nextRope)
-        state.nextRope = cur
-        return
-      case cur.kind
-      of RopeFgColor:
-        let tweak = FmtStyle(textColor: some(cur.color))
-        let style = state.curStyle.mergeStyles(tweak)
-        state.pushStyle(style)
-        subextract(r.toColor)
-        state.popStyle()
-      of RopeBgColor:
-        let tweak = FmtStyle(bgColor: some(cur.color))
-        state.pushStyle(state.curStyle.mergeStyles(tweak))
-        addStyledText(subextract(cur.toColor))
-        state.popStyle()
-      of RopeBreak:
-        # For now, we don't care about kind of break.
-        extract.lines.add(@[])
-      of RopeTaggedContainer:
-        let styleOpt = state.getNewStartStyle(cur)
-        state.pushStyle(styleOpt.getOrElse(state.curStyle))
-        subextract(cur.contained)
-        state.popStyle()
-      else:
-        assert false
-
-    cur = cur.next
-
-proc extractText(state: var FmtState, r: Rope): TextPlane =
-  result = TextPlane(lines: @[@[]])
-  state.extractText(r, result)
-
-proc preRenderTextBox(state: var FmtState, p: seq[TextPlane]): seq[RenderBox] =
-  state.boxContent(state.curStyle, result):
-    var merged = p.mergeTextPlanes()
-    merged.wrapToWidth(state.curStyle, state.totalWidth)
-    result = @[RenderBox(contents: merged, width: state.totalWidth)]
-
-template planesToBox() =
-  if len(consecutivePlanes) != 0:
-    result &= state.preRenderTextBox(consecutivePlanes)
-    consecutivePlanes = @[]
-
-proc preRenderAligned(state: var FmtState, r: Rope): seq[RenderBox] =
-  var tweak: Option[FmtStyle]
-
-  case r.tag[0]
-  of 'l':
-    tweak = some(FmtStyle(alignStyle: some(AlignL)))
-  of 'c':
-    tweak = some(FmtStyle(alignStyle: some(AlignC)))
-  of 'r':
-    tweak = some(FmtStyle(alignStyle: some(AlignR)))
-  of 'j':
-    tweak = some(FmtStyle(alignStyle: some(AlignJ)))
-  of 'f':
-    tweak = some(FmtStyle(alignStyle: some(AlignF)))
-  else:
-    discard
-
-  fmtBox(tweak):
-    result = state.preRender(r.contained)
-
-proc preRenderBreak(state: var FmtState, r: Rope): seq[RenderBox] =
-  standardBox:
-    result = state.preRender(r.guts)
-
-proc preRenderTagged(state: var FmtState, r: Rope): seq[RenderBox] =
-  standardBox:
-    result = state.preRender(r.contained)
+proc preRenderLink(state: var FmtState, r: Rope) =
+  if not r.noBoxRequired():
+    raise newException(ValueError, "Only styled text is allowed in links")
 
-proc preRenderColor(state:  var FmtState, r: Rope): seq[RenderBox] =
-  standardBox:
-    result = state.preRender(r.toColor)
+  discard state.preRender(r.toHighlight)
+  if state.showLinkTarg:
+    withRopeStyle:
+      addStyleMarkers:
+        let runes = @[Rune('(')] & r.url.toRunes() & @[Rune(')')]
+        addStyleMarkers:
+          state.curPlane.addRunesToExtraction(runes)
 
 proc preRender(state: var FmtState, r: Rope): seq[RenderBox] =
-  ## This version of prerender returns a COLUMN of boxes of one single
-  ## width.  But generally, there should only be one item in the
-  ## column when possible, which itself should consist of one
-  ## TextPlane item.
-  ##
-  ## The exception to that is RopeTableRows, which leaves it to
-  ## RopeTable to do the combination.
+  # This is the actual main worker for rendering. Generally, these
+  # ropes are trees that may be concatenated, so we need to go down
+  # first, and then over.
+  #
+  # When individual pieces of a tree have text, that text gets
+  # extracted, and is 'formatted' based on the style of the active
+  # style (as determined by parent containers). This basically means
+  # keeping track of the style and injecting start/end markers.
+  #
+  # Instead of injecting markers only when styles change, we currenly
+  # make life easy on ourselves, and wrap each text atom in
+  # markers. It's much easier to reason about this way.
+  #
+  # Every time we hit a box boundary, we commit the current text
+  # extraction (a TextPlane) and shove it in a RenderBox.
+  #
+  # Note that, because of contatenation that can link in above us, it
+  # would take some accounting for a node to know for sure whether
+  # it's going to be the last thing to write into a TextPlane, so we
+  # always check when we enter a container to see if we need to
+  # render, before we switch the active style.
 
-  var
-    consecutivePlanes: seq[TextPlane]
-    curRope = r
+  if r == nil:
+    return
 
-  while curRope != nil:
-    if curRope.noBoxRequired():
-      let textBox = state.extractText(curRope)
-      consecutivePlanes.add(textBox)
+  case r.kind
+  of RopeAtom:
+    state.preRenderAtom(r)
+  of RopeLink:
+    state.preRenderLink(r)
+  of RopeFgColor, RopeBgColor:
+    withRopeStyle:
+      result = state.preRender(r.toColor)
+  of RopeBreak:
+    if r.guts != nil:
+      # It's a 

or similar, so a box. + result.enterContainer: + result &= state.preRender(r.guts) else: - planesToBox() - - case curRope.kind - of RopeList: - if curRope.tag == "ul": - result &= state.preRenderUnorderedList(curRope) - else: - result &= state.preRenderOrderedList(curRope) - of RopeTable: - result &= state.preRenderTable(curRope) - of RopeTableRow: - result &= state.preRenderRow(curRope) - of RopeTableRows: - result &= state.preRenderRows(curRope) - of RopeAlignedContainer: - result &= state.preRenderAligned(curRope) - of RopeBreak: - result &= state.preRenderBreak(curRope) - of RopeTaggedContainer: - result &= state.preRenderTagged(curRope) - of RopeFgColor, RopeBgColor: - result &= state.preRenderColor(curRope) - else: - discard - - while curRope != nil: - if state.nextRope != nil: - curRope = state.nextRope - state.nextRope = state.savedRopes.pop() - else: - curRope = curRope.next - if curRope in state.processed: - continue + state.curPlane.lines.add(@[]) + of RopeTaggedContainer: + case r.tag + of "width": + # This is not going to be a tagged container forever. + withWidth(r.width): + result.enterContainer: + result &= state.preRender(r.contained) + of "colors": + # Also will stop being a tagged container. + let tmp = state.preRender(r.contained) + for item in tmp: + item.contents.wrapTextPlane(@[StyleColor], @[StyleColorPop]) + if state.curPlane.lines.len() > 1 or + (state.curPlane.lines.len() != 0 and + len(state.curPlane.lines[0]) != 0): + state.curPlane.wrapTextPlane(@[StyleColor], @[StyleColorPop]) + result &= tmp + of "nocolors": + # Also will stop being a tagged container. + let tmp = state.preRender(r.contained) + for item in tmp: + item.contents.wrapTextPlane(@[StyleNoColor], @[StyleColorPop]) + if state.curPlane.lines.len() > 1 or + (state.curPlane.lines.len() != 0 and + len(state.curPlane.lines[0]) != 0): + state.curPlane.wrapTextPlane(@[StyleNoColor], @[StyleColorPop]) + result &= tmp + else: + if r.isContainer(): + result.enterContainer: + applyContainerStyle(result): + result &= state.preRender(r.contained) else: - break - - planesToBox() + withRopeStyle: + result &= state.preRender(r.contained) + of RopeList: + result.enterContainer: + applyContainerStyle(result): + if r.tag == "ul": + result &= state.preRenderUnorderedList(r) + else: + result &= state.preRenderOrderedList(r) + of RopeTable: + result.enterContainer: + applyContainerStyle(result): + result &= state.preRenderTable(r) + of RopeTableRow: + result.enterContainer: + applyContainerStyle(result): + result &= state.preRenderRow(r) + of RopeTableRows: + result.enterContainer: + applyContainerStyle(result): + result &= state.preRenderRows(r) + + result &= state.preRender(r.next) proc preRender*(r: Rope, width = -1, showLinkTargets = false, - defaultStyle = defaultStyle): TextPlane = - ## Denoted in the stream of characters to output, what styles - ## should be applied, when. We do this by dropping in unique - ## values into the uint32 stream that cannot be codepoints. This - ## instructs the rendering implementation what style to push. - ## - ## There's a value for pop as well. + style = defaultStyle, outerPad = true): TextPlane = + ## This takes a Rope that is essentially stored as a tree annotated + ## with style information, and produce a representation that is an + ## array of lines of unicode characters, interspersed with 32-bit + ## values outside the range of Unicode codepoints that allow for + ## lookup of styling information from the tree. ## ## Note that if you don't pass a width in, we end up calling an ## ioctl to query the terminal width. That does seem a bit @@ -863,37 +825,33 @@ proc preRender*(r: Rope, width = -1, showLinkTargets = false, ## until this becomes a problem in the real world, we'll just ## leave it. - var - state = FmtState(curStyle: defaultStyle, showLinkTarg: showLinkTargets) - strip = false - + var state = FmtState(curStyle: style.mergeStyles(defaultStyle), + showLinkTarg: showLinkTargets, + curPlane: TextPlane(lines: @[])) if width <= 0: - state.totalWidth = terminalWidth() + state.totalWidth = terminalWidth() + width else: state.totalWidth = width - if state.totalWidth <= 0: - state.totalWidth = defaultTextWidth - if r.noBoxRequired(): - state.totalWidth = r.unboxedRuneLength() + 1 - strip = true + var + preRenderBoxes = state.preRender(r) + noBox: bool - let preRender = state.collapseColumn(state.preRender(r)) - result = state.collapsedBoxToTextPlane(preRender) - result.width = state.totalWidth + # If we had text linked at the end that started after the last container + # closed, then we will need to get that in here. We can check here to see + # if there were any breaks, and if there weren't, skip the collapse. + preRenderBoxes.applyContainerStyle: + if preRenderBoxes.len() == 0: + nobox = true - if strip: - var n = len(result.lines) - result.softBreak = true + if noBox: + return state.curPlane - while n != 0: - n -= 1 - result.lines[n] = result.lines[n].stripSpacesButNotFormattersFromEnd() - if result.lines[n].u32LineLength() != 0: - break - while len(result.lines) != 0: - if result.lines[^1].u32LineLength() == 0: - result.lines = result.lines[0 ..< ^1] - else: - break + if not outerPad: + preRenderBoxes[0].tmargin = 0 + preRenderBoxes[^1].bmargin = 0 + + let preRender = state.collapseColumn(preRenderBoxes) + result = state.collapsedBoxToTextPlane(preRender) + result.width = state.totalWidth diff --git a/nimutils/rope_styles.nim b/nimutils/rope_styles.nim index c739486..824fcb9 100644 --- a/nimutils/rope_styles.nim +++ b/nimutils/rope_styles.nim @@ -2,16 +2,60 @@ ## :Author: John Viega (john@crashoverride.com) ## :Copyright: 2023, Crash Override, Inc. -import unicode, tables, rope_base, options +import unicode, tables, rope_base, rope_construct, options, random, hexdump proc newStyle*(fgColor = "", bgColor = "", overflow = OIgnore, hang = -1, lpad = -1, rpad = -1, casing = CasingIgnore, tmargin = -1, bmargin = -1, bold = BoldIgnore, inverse = InverseIgnore, strikethru = StrikeThruIgnore, italic = ItalicIgnore, underline = UnderlineIgnore, - bulletChar = Rune(0x0000), minColWidth = -1, maxColWidth = -1, - borders: openarray[BorderOpts] = [], boxStyle: BoxStyle = nil, - align = AlignIgnore): FmtStyle = + bulletChar = Rune(0x0000), borders: openarray[BorderOpts] = [], + boxStyle: BoxStyle = nil, align = AlignIgnore): FmtStyle = + ## A swiss-army-knife interface for getting the exact style you're + ## looking for. + ## + ## Styles get pushed and popped while walking through the + ## underlying DOM; when multiple styles are pushed at once, + ## they're *merged*, with anything in the newer style overriding + ## anything it defines. + ## + ## For the parameters, the default values all mean "inherit from + ## the existing style". Anything you specify beyond the defaults + ## will lead to an override. + ## + ## Internally, individual style objects use Option[]s to know what + ## to override or not. + ## + ## Also, you need to consider the order in which style objects get + ## applied. There's usually a default style in place when we start + ## the pre-rendering process. + ## + ## Then, as we get to individual nodes, we look up the current + ## active style mapped to the for the node type (which map to + ## common html tags, and are interchangably called tags). If + ## there's a style, we merge it in (popping / unmerging it at the + ## end of processing the node). + ## + ## Then, if the node is in a 'class' (akin to an HTML class), and + ## there's a stored style for that class, we apply its overrides + ## next. + ## + ## Finally, if the node has an 'id' set, we apply any style + ## associated with the id. Then, we render. + ## + ## If you use the style API on rope objects, it works by mutating + ## the overrides, using the style associated w/ the node's ID + ## (creating a random ID if one isn't set). + ## + ## This means they take higher precidence than anything else, BUT, + ## the effects you set could still be masked by sub-nodes in the + ## tree. For instance, if you set something on a `table` node, the + ## saved styles for `tbody`, `tr` and `th`/`td` nodes can end up + ## masking what you were trying to set. + ## + ## The overall style for tags, classes and IDs can be set with + ## `setStyle()` and retrieved with `getStyle()`. + result = FmtStyle() if fgColor != "": @@ -20,7 +64,7 @@ proc newStyle*(fgColor = "", bgColor = "", overflow = OIgnore, hang = -1, result.bgColor = some(bgColor) if overflow != OIgnore: result.overFlow = some(overflow) - if hang >= 0: + if hang != -1: result.hang = some(hang) if lpad >= 0: result.lpad = some(lpad) @@ -64,46 +108,42 @@ proc newStyle*(fgColor = "", bgColor = "", overflow = OIgnore, hang = -1, result.underlineStyle = some(underline) if bulletChar != Rune(0x0000): result.bulletChar = some(bulletChar) - if minColWidth != -1: - result.minTableColWidth = some(minColWidth) - if maxColWidth != -1: - result.maxTableColWidth = some(maxColWidth) if len(borders) != 0: for item in borders: case item of BorderNone: - result.useTopBorder = some(false) - result.useBottomBorder = some(false) - result.useLeftBorder = some(false) - result.useRightBorder = some(false) - result.useVerticalSeparator = some(false) + result.useTopBorder = some(false) + result.useBottomBorder = some(false) + result.useLeftBorder = some(false) + result.useRightBorder = some(false) + result.useVerticalSeparator = some(false) result.useHorizontalSeparator = some(false) of BorderTop: - result.useTopBorder = some(true) + result.useTopBorder = some(true) of BorderBottom: - result.useBottomBorder = some(true) + result.useBottomBorder = some(true) of BorderLeft: - result.useLeftBorder = some(true) + result.useLeftBorder = some(true) of BorderRight: - result.useRightBorder = some(true) + result.useRightBorder = some(true) of HorizontalInterior: result.useHorizontalSeparator = some(true) of VerticalInterior: - result.useVerticalSeparator = some(true) + result.useVerticalSeparator = some(true) of BorderTypical: - result.useTopBorder = some(true) - result.useBottomBorder = some(true) - result.useLeftBorder = some(true) - result.useRightBorder = some(true) - result.useVerticalSeparator = some(true) + result.useTopBorder = some(true) + result.useBottomBorder = some(true) + result.useLeftBorder = some(true) + result.useRightBorder = some(true) + result.useVerticalSeparator = some(true) result.useHorizontalSeparator = some(false) of BorderAll: - result.useTopBorder = some(true) - result.useBottomBorder = some(true) - result.useLeftBorder = some(true) - result.useRightBorder = some(true) - result.useVerticalSeparator = some(true) + result.useTopBorder = some(true) + result.useBottomBorder = some(true) + result.useLeftBorder = some(true) + result.useRightBorder = some(true) + result.useVerticalSeparator = some(true) result.useHorizontalSeparator = some(true) if boxStyle != nil: result.boxStyle = some(boxStyle) @@ -111,68 +151,102 @@ proc newStyle*(fgColor = "", bgColor = "", overflow = OIgnore, hang = -1, result.alignStyle = some(align) var - defaultStyle* = newStyle(overflow = OWrap, lpad = 0, rpad = 0, tmargin = 0) - tableDefault = newStyle(borders = [BorderAll], overflow = OWrap, tmargin = 0, - fgColor = "white", bgcolor = "dodgerblue") - # 1. Even / odd columns - # 2. Table margins + plainStyle = FmtStyle( + textColor: some(""), bgColor: some(""), overflow: some(OWrap), + lpad: some(0), rpad: some(0), tmargin: some(0), bmargin: some(0), + casing: some(CasingIgnore), bold: some(false), hang: some(2), + inverse: some(false), strikethrough: some(false), + italic: some(false), underlineStyle: some(UnderlineIgnore), + useTopBorder: some(true), useLeftBorder: some(true), + useRightBorder: some(true), useVerticalSeparator: some(false), + useHorizontalSeparator: some(false), boxStyle: some(BoxStyleDouble), + alignStyle: some(AlignIgnore)) + + defaultStyle* = plainStyle + tableDefault = newStyle(overflow = OWrap, tmargin = 0, bmargin = 0, + lpad = 0, rpad = 0) + styleMap*: Table[string, FmtStyle] = { - "body" : newStyle(rpad = 1, lpad = 1), - "div" : newStyle(rpad = 1, lpad = 1, bgColor = "none"), - "p" : newStyle(bmargin = 1), - "h1" : newStyle(fgColor = "red", bold = BoldOn, - align = AlignL, italic = ItalicOn, casing = CasingUpper, - tmargin = 1, bmargin = 0), - "h2" : newStyle(fgColor = "lime", bgColor = "darkslategray", - bold = BoldOn, align = AlignL, italic = ItalicOn, tmargin = 2), - "h3" : newStyle(bgColor = "red", fgColor = "white", - italic = ItalicOn, tmargin = 1, casing = CasingUpper), - "h4" : newStyle(bgColor = "red", fgColor = "white", italic = ItalicOn, - underline = UnderlineSingle, casing = CasingTitle), - "h5" : newStyle(fgColor = "darkslategray", bgColor = "lime", - italic = ItalicOn, casing = CasingTitle), - "h6" : newStyle(fgColor = "yellow", bgColor = "blue", - underline = UnderlineSingle, casing = CasingTitle), - "ol" : newStyle(bulletChar = Rune('.'), lpad = 2, align = AlignL), - "ul" : newStyle(bulletChar = Rune(0x2022), lpad = 2, - align = AlignL), #• - "li" : newStyle(lpad = 1, overflow = OWrap, align = AlignL), - "table" : tableDefault, - "thead" : tableDefault, - "tbody" : tableDefault, - "tfoot" : tableDefault, - "tborder" : tableDefault, - "td" : newStyle(tmargin = 0, overflow = OWrap, align = AlignL, - lpad = 1, rpad = 1), - "th" : newStyle(bgColor = "black", bold = BoldOn, overflow = OWrap, - casing = CasingUpper, tmargin = 0, fgColor = "lime", - align = AlignC), - "tr" : newStyle(fgColor = "white", bold = BoldOn, lpad = 0, rpad = 0, - overflow = OWrap, tmargin = 0, bgColor = "dodgerblue"), - "tr.even" : newStyle(fgColor = "white", bgColor = "dodgerblue", - tmargin = 0, overflow = OWrap), - "tr.odd" : newStyle(fgColor = "white", bgColor = "steelblue", - tmargin = 0, overflow = OWrap), - "em" : newStyle(fgColor = "jazzberry", italic = ItalicOn), - "strong" : newStyle(inverse = InverseOn, italic = ItalicOn), - "code" : newStyle(inverse = InverseOn, italic = ItalicOn), - "caption" : newStyle(bgColor = "black", fgColor = "atomiclime", - align = AlignC, italic = ItalicOn, bmargin = 2) + "container" : newStyle(rpad = 1, lpad = 1), + "div" : newStyle(rpad = 1, lpad = 1, bgColor = "none"), + "p" : newStyle(bmargin = 0), + "basic" : newStyle(bmargin = 0), + "h1" : newStyle(fgColor = "red", bold = BoldOn, align = AlignL, + italic = ItalicOn, casing = CasingUpper, tmargin = 2), + "h2" : newStyle(fgColor = "lime", bgColor = "darkslategray", + tmargin = 1, + bold = BoldOn, align = AlignL, italic = ItalicOn), + "h3" : newStyle(bgColor = "red", fgColor = "white", tmargin = 1, + italic = ItalicOn, bmargin = 0, casing = CasingUpper), + "h4" : newStyle(bgColor = "red", fgColor = "white", tmargin = 1, + italic = ItalicOn, bmargin = 0, + underline = UnderlineSingle, casing = CasingTitle), + "h5" : newStyle(fgColor = "darkslategray", bgColor = "lime", + tmargin = 0, bmargin = 0, italic = ItalicOn, + casing = CasingTitle), + "h6" : newStyle(fgColor = "yellow", bgColor = "blue", tmargin = 0, + bmargin = 0, underline = UnderlineSingle, casing = CasingTitle), + "ol" : newStyle(bulletChar = Rune('.'), lpad = 2, align = AlignL, + tmargin = 0, bmargin = 0), + "ul" : newStyle(bulletChar = Rune(0x2022), lpad = 2, + tmargin = 1, bmargin = 1, align = AlignL), #• + "li" : newStyle(lpad = 1, overflow = OWrap, align = AlignL, + tmargin = 0, bmargin = 0), + "left" : newStyle(align = AlignL), + "right" : newStyle(align = AlignR), + "center" : newStyle(align = AlignC), + "justify" : newStyle(align = AlignJ), + "flush" : newStyle(align = AlignF), + "table" : newStyle(borders = [BorderAll], tmargin = 1, bmargin = 1, + lpad = 1, rpad = 1, bgColor = "dodgerblue"), + "thead" : tableDefault, + "tbody" : tableDefault, + "tfoot" : tableDefault, + "plain" : plainStyle, + "text" : defaultStyle, + "td" : newStyle(tmargin = 0, bmargin = 0, overflow = OWrap, + align = AlignL, lpad = 1, rpad = 1), + "th" : newStyle(bgColor = "black", bold = BoldOn, overflow = OWrap, + casing = CasingUpper, tmargin = 0, bmargin = 0, fgColor = "lime", + lpad = 1, rpad = 1, align = AlignC), + "tr" : newStyle(fgColor = "white", bold = BoldOn, lpad = 0, rpad = 0, + overflow = OWrap, tmargin = 0, bmargin = 0, + bgColor = "dodgerblue"), + "tr.even" : newStyle(fgColor = "white", bgColor = "slategray", + tmargin = 0, overflow = OWrap), + "tr.odd" : newStyle(fgColor = "white", bgColor = "steelblue", + tmargin = 0, overflow = OWrap), + "em" : newStyle(fgColor = "jazzberry", italic = ItalicOn), + "italic" : newStyle(italic = ItalicOn), + "i" : newStyle(italic = ItalicOn), + "u" : newStyle(underline = UnderlineSingle), + "underline" : newStyle(underline = UnderlineSingle), + "strong" : newStyle(inverse = InverseOn, italic = ItalicOn), + "code" : newStyle(inverse = InverseOn, italic = ItalicOn), + "caption" : newStyle(bgColor = "black", fgColor = "atomiclime", + lpad = 2, rpad = 3, + tmargin = 0, bmargin = 0, align = AlignC, italic = ItalicOn) }.toTable() - perClassStyles* = Table[string, FmtStyle]() + perClassStyles*: Table[string, FmtStyle] = { + "callout" : newStyle(fgColor = "fandango", bgColor = "jazzberry", + italic = ItalicOn, casing = CasingTitle) + }.toTable() + perIdStyles* = Table[string, FmtStyle]() breakingStyles*: Table[string, bool] = { + "container" : true, + "basic" : true, "caption" : true, + "code" : true, "p" : true, "div" : true, "ol" : true, "ul" : true, "li" : true, "blockquote" : true, - "pre" : true, "q" : true, "small" : true, "td" : true, @@ -183,7 +257,12 @@ var "h3" : true, "h4" : true, "h5" : true, - "h6" : true + "h6" : true, + "left" : true, + "right" : true, + "center" : true, + "justify" : true, + "flush" : true }.toTable() {.emit: """ @@ -211,6 +290,14 @@ var styleToIdMap: Table[FmtStyle, uint32] proc getStyleId*(s: FmtStyle): uint32 = + ## This is really meant to be an internal API, but cross-module. It + ## returns a 32-bit integer unique to the style (across the life of + ## program execution). This integer is added to the pre-render + ## stream to pass on information to the renderer, and is explicitly + ## NOT in the range of valid unicode characters. + ## + ## Note that the renderer does not need to keep stack state; the + ## styles it looks up have had all stack operations pre-applied. if s in styleToIdMap: return styleToIdMap[s] @@ -220,6 +307,9 @@ proc getStyleId*(s: FmtStyle): uint32 = styleToIdMap[s] = result proc idToStyle*(n: uint32): FmtStyle = + ## The inverse of `getStyleId()`, which enables renderers to look up + ## style information associated with the text that follows (until + ## seeing a reset marker or another style ID.) result = idToStyleMap[n] if result == defaultStyle: @@ -229,12 +319,18 @@ proc idToStyle*(n: uint32): FmtStyle = return proc mergeStyles*(base: FmtStyle, changes: FmtStyle): FmtStyle = + ## This layers any new overrides on top of existing values, creating + ## a third style. result = base.copyStyle() - result.lpad = changes.lpad - result.rpad = changes.rpad - result.tmargin = changes.tmargin - result.bmargin = changes.bmargin + if changes.lpad.isSome(): + result.lpad = changes.lpad + if changes.rpad.isSome(): + result.rpad = changes.rpad + if changes.tmargin.isSome(): + result.tmargin = changes.tmargin + if changes.bmargin.isSome(): + result.bmargin = changes.bmargin if changes.textColor.isSome(): result.textColor = changes.textColor if changes.bgColor.isSome(): @@ -257,10 +353,6 @@ proc mergeStyles*(base: FmtStyle, changes: FmtStyle): FmtStyle = result.underlineStyle = changes.underlineStyle if changes.bulletChar.isSome(): result.bulletChar = changes.bulletChar - if changes.minTableColWidth.isSome(): - result.minTableColWidth = changes.minTableColWidth - if changes.maxTableColWidth.isSome(): - result.maxTableColWidth = changes.maxTableColWidth if changes.useTopBorder.isSome(): result.useTopBorder = changes.useTopBorder if changes.useBottomBorder.isSome(): @@ -278,13 +370,480 @@ proc mergeStyles*(base: FmtStyle, changes: FmtStyle): FmtStyle = if changes.alignStyle.isSome(): result.alignStyle = changes.alignStyle +proc isContainer*(r: Rope): bool = + ## Returns true if the rope itself represents a "container", meaning + ## a layout box. + if r == nil: + return false + + case r.kind + of RopeAtom, RopeLink, RopeFgColor, RopeBgColor: + return false + of RopeList, RopeTable, RopeTableRow, RopeTableRows: + return true + of RopeBreak: + if r.guts == nil: + return false + else: + return true + of RopeTaggedContainer: + if r.tag in breakingStyles or r.noTextExtract: + return true + else: + return false + template setDefaultStyle*(style: FmtStyle) = + ## This call allows you to set the default starting style, which is + ## applied whenever you call a routine that formats (such as + ## `print()` or `stylize()` defaultStyle = style -type StyleType = enum StyleTypeTag, StyleTypeClass, StyleTypeId +type StyleType* = enum StyleTypeTag, StyleTypeClass, StyleTypeId proc setStyle*(reference: string, style: FmtStyle, kind = StyleTypeTag) = + ## Set a specific output style for a tag, class or ID, based on the + ## value of the `kind` parameter. See the notes on `newStyle()` for + ## how this gets applied. case kind of StyleTypeTag: styleMap[reference] = style of StyleTypeClass: perClassStyles[reference] = style of StyleTypeId: perIdStyles[reference] = style + +proc getStyle*(reference: string, kind = StyleTypeTag): FmtStyle = + ## Return the installed style associated w/ a Tag, Class or Id. + return case kind + of StyleTypeTag: + if reference == "default": + return defaultStyle + else: + styleMap[reference] + of StyleTypeClass: perClassStyles[reference] + of StyleTypeId: perIdStyles[reference] + +proc setClass*(r: Rope, name: string, recurse = false) = + ## Sets the 'class' associated with a given Rope. + var toProcess: seq[Rope] + + if recurse: + toProcess = r.ropeWalk() + else: + toProcess.add(r) + + for item in toProcess: + item.class = name + +proc setID*(r: Rope, name: string) = + ## Sets the 'id' associated with a given rope manually. + +proc findFirstContainer*(r: Rope): Rope = + ## If a rope has annotation nodes at the top, skip them to find + ## the first container node, or return null if not. + for item in r.ropeWalk(): + if item.isContainer(): + return item + +proc ropeStyle*(r: Rope, + style: FmtStyle, + recurse = false, + container = false): Rope + {.discardable.} = + ## Edits the style for a specific rope, merging any passed overrides + ## into the style associated with the Rope's ID. + ## + ## If the rope has no ID, it is assigned one at random by + ## hex-encoding a 64-bit random value. + + var toProcess: seq[Rope] + + result = r + + if r == nil: + return + + if container and not r.isContainer(): + return r.findFirstContainer().ropeStyle(style, true, recurse) + + if recurse: + toProcess = r.ropeWalk() + else: + toProcess.add(r) + + for item in toProcess: + if container: + if not item.isContainer(): + continue + item.noTextExtract = true + + if item.id == "": + item.id = randString(8).hex() + + if item.id in perIdStyles: + perIdStyles[item.id] = perIdStyles[item.id].mergeStyles(style) + else: + perIdStyles[item.id] = style + +proc colPcts*(r: Rope, pcts: openarray[int]): Rope {.discardable.} = + if r == nil: + return + + var info: seq[ColInfo] + for item in pcts: + info.add(ColInfo(span: 1, widthPct: item)) + + for item in r.search(tag = ["table"]): + item.colInfo = info + + return r + +proc applyClass*(r: Rope, class: string, recurse = true): Rope {.discardable.} = + if r == nil: + return + + if recurse: + for item in r.ropeWalk(): + item.class = class + else: + r.class = class + +proc noBorders*(r: Rope, recurse = true): Rope {.discardable.} = + ## Overrides a rope's current style to remove any + ## table borders. + return r.ropeStyle(newStyle(borders = [BorderNone]), recurse, true) + +proc allBorders*(r: Rope, recurse = true): Rope {.discardable.} = + ## Overrides a rope's current style to add all + ## table borders. + return r.ropeStyle(newStyle(borders = [BorderAll]), recurse, true) + +proc typicalBorders*(r: Rope, recurse = true): Rope {.discardable.} = + ## Overrides a rope's current style to set 'typical' + ## table borders, which is all except for internal + ## horizontal separators. + return r.ropeStyle(newStyle(borders = [BorderTypical]), recurse, true) + +proc defaultBg*(r: Rope, recurse = true): Rope {.discardable.} = + ## Remove any background color for the current node (going with + ## whatever the environment uses by default). Note that it may not + ## have the effect you think if sub-nodes set a color (i.e., don't + ## do this at the top level of a table). + return r.ropeStyle(FmtStyle(bgColor: some("")), recurse) + +proc defaultBg*(s: string): Rope {.discardable.} = + ## Return a Rope that will ensure the current string's background + ## color is not set (unless you later change it with another call). + ## + ## This call does NOT process embedded markdown. + return pre(s).defaultBg() + +proc defaultFg*(r: Rope, recurse = true): Rope {.discardable.} = + ## Remove the foreground text color for the current node (applying + ## whatever the environment uses by default). Note that it may not + ## have the effect you think if sub-nodes set a color (i.e., don't + ## do this at the top level of a table). + return r.ropeStyle(FmtStyle(textColor: some("")), recurse) + +proc defaultFg*(s: string): Rope {.discardable.} = + ## Return a Rope that will ensure the current string's foreground + ## color is not set (unless you later change it with another call). + ## + ## This call does NOT process embedded markdown. + return pre(s).defaultFg() + +proc bgColor*(r: Rope, color: string, recurse = true): Rope {.discardable.} = + ## Overrides a rope's current style to set the background color. + ## Note that sub-nodes will still have their style applied after + ## this, and can have an impact. For instance, setting this at the + ## "table" level is unlikely to affect the entire table. + return r.ropeStyle(newStyle(bgColor = color), recurse) + +proc bgColor*(s: string, color: string): Rope {.discardable.} = + ## Returns a new Rope from a string, where the node will have the + ## explicit background color applied. This will not have sub-nodes, + ## so should override other settings (e.g., in a table). + return pre(s).bgColor(color) + +proc fgColor*(r: Rope, color: string, recurse = true): Rope {.discardable.} = + ## Overrides a rope's current style to set the foreground color. + ## Note that sub-nodes will still have their style applied after + ## this, and can have an impact. For instance, setting this at the + ## "table" level is unlikely to affect the entire table. + return r.ropeStyle(newStyle(fgColor = color), recurse) + +proc fgColor*(s: string, color: string): Rope {.discardable.} = + ## Returns a new Rope from a string, where the node will have the + ## explicit foreground color applied. This will not have sub-nodes, + ## so should override other settings (e.g., in a table). + return pre(s).fgColor(color) + +proc topMargin*(r: Rope, n: int): Rope {.discardable.} = + ## Add a top margin to a Rope object. This will be ignored if the + ## rope isn't a 'block' of some sort. + result = r + var tableSet = r.search("table", first = true) + if tableSet.len() == 1: + tableSet[0].ropeStyle(newStyle(tmargin = n), false) + +proc topMargin*(s: string, n: int): Rope {.discardable.} = + ## Adds a top margin to a string. + return pre(s).topMargin(n) + +proc bottomMargin*(r: Rope, n: int): Rope {.discardable.} = + ## Add a bottom margin to a Rope object. This will be ignored if the + ## rope isn't a 'block' of some sort. + result = r + var tableSet = r.search("table", first = true) + if tableSet.len() == 1: + tableSet[0].ropeStyle(newStyle(bmargin = n), false) + +proc bottomMargin*(s: string, n: int): Rope {.discardable.} = + ## Adds a bottom margin to a string. + return pre(s).bottomMargin(n) + +proc leftPad*(r: Rope, n: int, recurse = false): Rope {.discardable.} = + ## Add left padding to a Rope object. This will be ignored if the + ## rope isn't a 'block' of some sort. + return r.ropeStyle(newStyle(lpad = n), recurse, true) + +proc leftPad*(s: string, n: int): Rope {.discardable.} = + ## Add left padding to a string. + result = pre(s).leftPad(n) + +proc rightPad*(r: Rope, n: int, recurse = false): Rope {.discardable.} = + ## Add right padding to a Rope object. This will be ignored if the + ## rope isn't a 'block' of some sort. + return r.ropeStyle(newStyle(rpad = n), recurse, true) + +proc rightPad*(s: string, n: int): Rope {.discardable.} = + ## Add right padding to a string. + result = s.text().rightPad(n) + +proc lpad*(r: string | Rope, n: int, recurse = false): Rope {.discardable.} = + ## Alias for `leftPad` + result = r.leftPad(n, recurse) + +proc rpad*(r: string | Rope, n: int): Rope {.discardable.} = + ## Alias for `rightPad` + result = r.rightPad(n) + +proc plainBorders*(r: Rope, recurse = true): Rope {.discardable.} = + ## Overrides any settings for table borders; any nested table + ## contents will have plain borders, *if* borders are set, and if + ## the border setting isn't overriden by an internal node. + return r.ropeStyle(newStyle(boxStyle = BoxStylePlain), recurse, true) + +proc boldBorders*(r: Rope, recurse = true): Rope {.discardable.} = + ## Overrides any settings for table borders; any nested table + ## contents will have bold borders, *if* borders are set, and if + ## the border setting isn't overriden by an internal node. + return r.ropeStyle(newStyle(boxStyle = BoxStyleBold), recurse, true) + +proc doubleBorders*(r: Rope, recurse = true): Rope {.discardable.} = + ## Overrides any settings for table borders; any nested table + ## contents will have double-lined borders, *if* borders are set, + ## and if the border setting isn't overriden by an internal node. + return r.ropeStyle(newStyle(boxStyle = BoxStyleDouble), recurse, true) + +proc dashBorders*(r: Rope, recurse = true): Rope {.discardable.} = + ## Overrides any settings for table borders; any nested table + ## contents will have dashed borders, *if* borders are set, + ## and if the border setting isn't overriden by an internal node. + ## + ## NB, All dashed borders don't render on all terminals, and we do + ## not currently attempt to detect this condition. + return r.ropeStyle(newStyle(boxStyle = BoxStyleDash), recurse, true) + +proc altDashBorders*(r: Rope, recurse = true): Rope {.discardable.} = + ## Overrides any settings for table borders; any nested table + ## contents will have an dashed borders created from an alternate + ## part of the Unicode character set. This is only applied *if* + ## borders are set, and if the border setting isn't overriden by an + ## internal node. + ## + ## NB, All dashed borders don't render on all terminals, and we do + ## not currently attempt to detect this condition. + return r.ropeStyle(newStyle(boxStyle = BoxStyleDash2), recurse, true) + +proc boldDashBorders*(r: Rope, recurse = true): Rope {.discardable.} = + ## Overrides any settings for table borders; any nested table + ## contents will have BOLD dashed borders, *if* borders are set, + ## and if the border setting isn't overriden by an internal node. + ## + ## NB, All dashed borders don't render on all terminals, and we do + ## not currently attempt to detect this condition. + return r.ropeStyle(newStyle(boxStyle = BoxStyleBoldDash), recurse, true) + +proc altBoldDashBorders*(r: Rope, recurse = true): Rope {.discardable.} = + ## Overrides any settings for table borders; any nested table + ## contents will have BOLD dashed borders created from an alternate + ## part of the Unicode character set. This is only applied *if* + ## borders are set, and if the border setting isn't overriden by an + ## internal node. + ## + ## NB, All dashed borders don't render on all terminals, and we do + ## not currently attempt to detect this condition. + return r.ropeStyle(newStyle(boxStyle = BoxStyleBoldDash2), recurse, true) + +proc boxStyle*(r: Rope, s: BoxStyle, recurse = true): Rope {.discardable.} = + ## Style box borders using a specific passed box style. Built-in + ## default values are: + ## + ## BoxStylePlain, BoxStyleBold, BoxStyleDouble, BoxStyleDash, + ## BoxStyleDash2, BoxStyleBoldDash, BoxStyleBoldDash2, + ## BoxStyleAsterisk, BoxStyleAscii + ## + ## Styles depend on the font having approrpiate unicode glyphs; + ## we've found the dashed boxes are less widely available. + return r.ropeStyle(newStyle(boxStyle = s), recurse, true) + +proc setBorders*(r: Rope, o: BorderOpts, recurse = true): Rope {.discardable.} = + ## Set which borders will be displayed for any tables within a rope. + return r.ropeStyle(newStyle(borders = [o]), recurse, true) + +proc setBullet*(r: Rope, c: Rune, recurse = true): Rope {.discardable.} = + ## Sets the character used as a bullet for lists. For ordered lists, + ## this does NOT change the numbering style, it changes the + ## character put after the number. Currently, there's no ability to + ## tweak the numbering style. You can instead use an unordered list, + ## remove the bullet with `removeBullet` and manually number. + if r != nil: + r.noTextExtract = true + return r.ropeStyle(newStyle(bulletChar = c), recurse) + +proc removeBullet*(r: Rope, recurse = true): Rope {.discardable.} = + ## Causes unordered list items to render without a bullet. + ## Ordered list items still render numbered; they just lose + ## the period after the number. + return r.ropeStyle(FmtStyle(bulletChar: some(Rune(0x0000))), recurse, true) + +proc setCasing*(r: Rope, casing: TextCasing, recurse = true): Rope + {.discardable.} = + ## Sets casing on a rope. + return r.ropeStyle(newStyle(casing = casing), recurse) + +proc setCasing*(s: string, casing: TextCasing): Rope = + return pre(s).setCasing(casing) + +proc setOverflow*(r: Rope, overflow: OverflowPreference, recurse = true): Rope + {.discardable.} = + ## Sets the overflow strategy. If you set OIntentWrap then you can + ## change the wrap hang value via `setHang()`. + return r.ropeStyle(newStyle(overflow = overflow), recurse) + +proc setOverflow*(s: string, overflow: OverflowPreference): Rope = + return pre(s).setOverflow(overflow) + +proc setHang*(r: Rope, hang: int, recurse = true): Rope {.discardable.} = + ## Sets the indent hang for when OIndentWrap is on. + return r.ropeStyle(newStyle(hang = hang), recurse) + +proc bold*(r: Rope, disable = false, recurse = true): Rope {.discardable.} = + var param: BoldPref + + if disable: + param = BoldOff + else: + param = BoldOn + + return r.ropeStyle(newStyle(bold = param), recurse) + +proc bold*(s: string): Rope = + return pre(s).bold() + +proc inverse*(r: Rope, disable = false, recurse = true): Rope {.discardable.} = + var param: InversePref + + if disable: + param = InverseOff + else: + param = InverseOn + + return r.ropeStyle(newStyle(inverse = param), recurse) + +proc inverse*(s: string): Rope = + return pre(s).inverse() + +proc strikethrough*(r: Rope, disable = false, recurse = true): + Rope {.discardable.} = + var param: StrikeThruPref + + if disable: + param = StrikeThruOff + else: + param = StrikeThruOn + + return r.ropeStyle(newStyle(strikethru = param), recurse) + +proc strikethrough*(s: string): Rope = + return pre(s).strikethrough() + +proc italic*(r: Rope, disable = false, recurse = true): Rope {.discardable.} = + var param: ItalicPref + + if disable: + param = ItalicOff + else: + param = ItalicOn + + return r.ropeStyle(newStyle(italic = param), recurse) + +proc italic*(s: string): Rope = + return pre(s).italic() + +proc underline*(r: Rope, kind = UnderlineSingle, recurse = true): + Rope {.discardable.} = + + return r.ropeStyle(newStyle(underline = kind), recurse) + +proc underline*(s: string): Rope = + return pre(s).underline() + +proc align*(r: Rope, kind: AlignStyle, recurse = false): Rope {.discardable.} = + ## Sets alignment preference for a rope as specified. Does NOT + ## recurse into children by default. + return r.ropeStyle(newStyle(align = kind), recurse) + +proc align*(s: string, kind: AlignStyle): Rope = + return pre(s).align(kind) + +proc center*(r: Rope, recurse = false): Rope {.discardable.} = + ## Centers a node. Unless recurse is on, this will only set the + ## preference on the top rope, not any children. + return r.ropeStyle(newStyle(align = AlignC), recurse) + +proc center*(s: string): Rope = + return pre(s).center() + +proc right*(r: Rope, recurse = false): Rope {.discardable.} = + ## Right-justifies a node. Unless recurse is on, this will only set + ## the preference on the top rope, not any children. + return r.ropeStyle(newStyle(align = AlignR), recurse) + +proc right*(s: string): Rope = + return pre(s).right() + +proc left*(r: Rope, recurse = false): Rope {.discardable.} = + ## Left-justifies a node. Unless recurse is on, this will only set + ## the preference on the top rope, not any children. + return r.ropeStyle(newStyle(align = AlignL), recurse) + +proc left*(s: string): Rope = + return pre(s).left() + +proc justify*(r: Rope, recurse = false): Rope {.discardable.} = + ## Does line justification (adds spaces between words to justify + ## both on the left and the right), except for any trailing + ## line. Unless recurse is on, this will only set the preference on + ## the top rope, not any children. + return r.ropeStyle(newStyle(align = AlignJ), recurse) + +proc justify*(s: string): Rope = + return pre(s).justify() + +proc flushJustify*(r: Rope, recurse = false): Rope {.discardable.} = + ## Does full flush line justification, including for the final line + ## (unless it is a single word). Unless recurse is on, this will + ## only set the preference on the top rope, not any children. + return r.ropeStyle(newStyle(align = AlignF), recurse) + +proc flushJustify*(s: string): Rope = + return pre(s).justify() diff --git a/nimutils/sha.nim b/nimutils/sha.nim index c12e3b5..0f29efd 100644 --- a/nimutils/sha.nim +++ b/nimutils/sha.nim @@ -16,15 +16,15 @@ proc sha256_raw(s: cstring, count: csize_t, md_buf: cstring) {.cdecl, dynLib:DLL proc sha512_raw(s: cstring, count: csize_t, md_buf: cstring) {.cdecl, dynLib:DLLUtilName,importc: "SHA512".} type - Sha256ctx = object + Sha256ctx* = object evpCtx: EVP_MD_CTX - Sha512ctx = object + Sha512ctx* = object evpCtx: EVP_MD_CTX - Sha3ctx = object + Sha3ctx* = object evpCtx: EVP_MD_CTX - Sha256Digest = array[32, uint8] - Sha512Digest = array[64, uint8] - Sha3Digest = array[64, uint8] + Sha256Digest* = array[32, uint8] + Sha512Digest* = array[64, uint8] + Sha3Digest* = array[64, uint8] proc `=destroy`*(ctx: Sha256ctx) = EVP_MD_CTX_free(ctx.evpCtx) @@ -36,27 +36,34 @@ proc `=destroy`*(ctx: Sha3ctx) = EVP_MD_CTX_free(ctx.evpCtx) proc initSha256*(ctx: var Sha256CTX) = + ## Initialize a streaming SHA256 hashing context. ctx.evpCtx = EVP_MD_CTX_new() discard EVP_Digest_Init_ex2(ctx.evpCtx, EVP_sha256(), nil) proc initSha512*(ctx: var Sha512CTX) = + ## Initialize a streaming SHA512 hashing context. ctx.evpCtx = EVP_MD_CTX_new() discard EVP_Digest_Init_ex2(ctx.evpCtx, EVP_sha512(), nil) proc initSha3*(ctx: var Sha3CTX) = + ## Initialize a streaming SHA3 hashing context. ctx.evpCtx = EVP_MD_CTX_new() discard EVP_Digest_Init_ex2(ctx.evpCtx, EVP_sha3_512(), nil) proc initSha256*(): Sha256Ctx = + ## Return a new streaming SHA256 hashing context. initSha256(result) proc initSha512*(): Sha512Ctx = + ## Return a new streaming SHA512 hashing context. initSha512(result) proc initSha3*(): Sha3Ctx = + ## Return a new streaming SHA3 hashing context. initSha3(result) proc update*(ctx: var (Sha256ctx|Sha512ctx|Sha3ctx), data: openarray[char]) = + ## Add data to the current hash context. if len(data) != 0: discard EVP_DigestUpdate(ctx.evpCtx, addr data[0], cuint(len(data))) @@ -76,21 +83,30 @@ proc finalRaw(ctx: var Sha3ctx): Sha3Digest = discard EVP_DigestFinal_ex(ctx.evpCtx, addr result[0], addr i) proc final*(ctx: var Sha256ctx): string = + ## Retrieve the raw binary output for the current hash context. + ## Use `finalHex()` for something more readable. tagToString(finalRaw(ctx)) proc final*(ctx: var Sha512ctx): string = + ## Retrieve the raw binary output for the current hash context. + ## Use `finalHex()` for something more readable. tagToString(finalRaw(ctx)) proc final*(ctx: var Sha3ctx): string = + ## Retrieve the raw binary output for the current hash context. + ## Use `finalHex()` for something more readable. tagToString(finalRaw(ctx)) template finalHex*[T](ctx: var T): string = + ## Retrieves the final hash value, as a hex string. final(ctx).toHex().toLowerAscii() proc `$`*(tag: Sha256Digest|Sha512Digest|Sha3Digest): string = tagToString(tag) proc sha256*(s: string): string = + ## A one-shot SHA-256 interface that outputs a raw value. + ## Call .hex() on it to convert to hex. var raw: Sha256Digest sha256_raw(cstring(s), csize_t(s.len()), cast[cstring](addr raw)) @@ -98,6 +114,8 @@ proc sha256*(s: string): string = return tagToString(raw) proc sha512*(s: string): string = + ## A one-shot SHA-512 interface that outputs a raw value. + ## Call .hex() on it to convert to hex. var raw: Sha512Digest sha512_raw(cstring(s), csize_t(s.len()), cast[cstring](addr raw)) @@ -105,6 +123,8 @@ proc sha512*(s: string): string = return tagToString(raw) proc sha3*(s: string): string = + ## A one-shot SHA-3 interface that outputs a raw value. + ## Call .hex() on it to convert to hex. var ctx: Sha3ctx initSha3(ctx) @@ -112,15 +132,22 @@ proc sha3*(s: string): string = return ctx.final() template sha256Hex*(s: string): string = + ## Deprecated. use s.sha256().hex() sha256(s).toHex().toLowerAscii() template sha512Hex*(s: string): string = + ## Deprecated. use s.sha512().hex() sha512(s).toHex().toLowerAscii() template sha3Hex*(s: string): string = + ## Deprecated. use s.sha3().hex() sha3(s).toHex().toLowerAscii() proc hmacSha256*(key: string, s: string): string = + ## Computes HMAC-SHA256 for a message. + ## + ## This returns the raw binary hash; Use .hex() to convert to + ## something human readable. var tag: Sha256Digest i: cuint @@ -130,6 +157,10 @@ proc hmacSha256*(key: string, s: string): string = result = tagToString(tag) proc hmacSha512*(key: string, s: string): string = + ## Computes HMAC-SHA512 for a message. + ## + ## This returns the raw binary hash; Use .hex() to convert to + ## something human readable. var tag: Sha512Digest i: cuint @@ -138,6 +169,10 @@ proc hmacSha512*(key: string, s: string): string = result = tagToString(tag) proc hmacSha3*(key: string, s: string): string = + ## Computes HMAC-SHA3 for a message. + ## + ## This returns the raw binary hash; Use .hex() to convert to + ## something human readable. var tag: Sha3Digest i: cuint @@ -147,22 +182,13 @@ proc hmacSha3*(key: string, s: string): string = result = tagToString(tag) template hmacSha256Hex*(key: string, s: string): string = + ## Deprecated. use .hex() hmacSha256(key, s).toHex().toLowerAscii() template hmacSha512Hex*(key: string, s: string): string = + ## Deprecated. use .hex() hmacSha512(key, s).toHex().toLowerAscii() template hmacSha3Hex*(key: string, s: string): string = + ## Deprecated. use .hex() hmacSha3(key, s).toHex().toLowerAscii() - -when isMainModule: - import streams - let text = newFileStream("logging.nim").readAll() - var ctx: Sha256ctx - - initSha256(ctx) - ctx.update(text) - echo ctx.final().toHex().toLowerAscii() - echo Sha256(text).toHex().toLowerAscii() - echo hmacSha256("foo", "bar").toHex().toLowerAscii() - echo hmacSha256Hex("foo", "bar") diff --git a/nimutils/sinks.nim b/nimutils/sinks.nim index 775d8c4..663b28b 100644 --- a/nimutils/sinks.nim +++ b/nimutils/sinks.nim @@ -3,7 +3,7 @@ import streams, tables, options, os, strutils, std/[net, uri, httpclient], s3client, pubsub, misc, random, encodings, std/tempfiles, - parseutils, unicodeid, openssl, file, std/asyncfutures + parseutils, openssl, file, std/asyncfutures const defaultLogSearchPath = @["/var/log/", "~/.log/", "."] @@ -411,7 +411,7 @@ proc postSinkOut(msg: string, cfg: SinkConfig, t: Topic, ignored: StringTable) = if uri.scheme == "https": context = newContext(verifyMode = CVerifyPeer) if pinnedCert != "": - discard context.context.SSL_CTX_load_verify_file(pinnedCert) + discard context.context.SSL_CTX_load_verify_file(cstring(pinnedCert)) client = newHttpClient(sslContext=context, timeout=timeout) else: if "disallow_http" in cfg.params: diff --git a/nimutils/subproc.nim b/nimutils/subproc.nim index c937145..141ef08 100644 --- a/nimutils/subproc.nim +++ b/nimutils/subproc.nim @@ -1,7 +1,7 @@ import switchboard, posix, random, os, file {.warning[UnusedImport]: off.} -{.compile: joinPath(splitPath(currentSourcePath()).head, "subproc.c").} +{.compile: joinPath(splitPath(currentSourcePath()).head, "c/subproc.c").} {.pragma: sproc, cdecl, importc, nodecl.} type @@ -53,9 +53,9 @@ type SPIoNone = 0, SpIoStdin = 1, SpIoStdout = 2, SpIoInOut = 3, SpIoStderr = 4, SpIoInErr = 5, SpIoOutErr = 6, SpIoAll = 7 - SPResultObj* {. importc: "sb_result_t", header: joinPath(splitPath(currentSourcePath()).head, "switchboard.h") .} = object + SPResultObj* {. importc: "sb_result_t", header: "switchboard.h" .} = object SPResult* = ptr SPResultObj - SubProcess* {.importc: "subprocess_t", header: joinPath(splitPath(currentSourcePath()).head, "switchboard.h") .} = object + SubProcess* {.importc: "subprocess_t", header: "switchboard.h" .} = object proc tcgetattr*(fd: cint, info: var Termcap): cint {. cdecl, importc, header: "", discardable.} @@ -78,67 +78,226 @@ proc subproc_get_signal(ctx: var SubProcess, wait: bool): cint {.sproc.} # Functions we can call directly w/o a nim proxy. proc setParentTermcap*(ctx: var SubProcess, tc: var Termcap) {.cdecl, importc: "subproc_set_parent_termcap", nodecl.} + ## Set the parent's termcap at the time of a fork, for when subprocesses + ## are using a pseudo-terminal (pty). Generally the default should be + ## good. + ## + ## If not provided, the parent will assume that it's going to proxy + ## most things from the child, and turn off any appropriate + ## functionality (such as character echo). + ## + ## This must be called before spawning a process. + proc setChildTermcap*(ctx: var SubProcess, tc: var Termcap) {.cdecl, importc: "subproc_set_parent_termcap", nodecl.} + ## Set the child's termcap at the time of a fork, for when subprocesses + ## are using a pseudo-terminal (pty). + ## + ## This does not need to be called unless customization is + ## necessary, but if you do call it, you must do so before spawning + ## a process. proc setPassthroughRaw*(ctx: var SubProcess, which: SPIoKind, combine: bool) {.cdecl, importc: "subproc_set_passthrough", nodecl.} + ## Low-level wrapper used by `setPassthrough()` + template setPassthrough*(ctx: var SubProcess, which = SPIoAll, merge = false) = + ## This controls how input from the user gets forwarded to the child + ## process. Currently, it can only be called before spawning a process. + ## + ## The streams denoted in the `which` parameter will have forwarding + ## enabled. ctx.setPassthroughRaw(which, merge) proc setCaptureRaw*(ctx: var SubProcess, which: SPIoKind, combine: bool) {.cdecl, importc: "subproc_set_capture", nodecl.} + ## The low-level interface used by `setCapture` + template setCapture*(ctx: var SubProcess, which = SPIoOutErr, merge = false) = + ## This controls what input streams will be *captured*. Captured + ## input is available to the user after the process ends, by calling + ## `getStdout()`, `getStderr()` and `getStdin()` on the `SPResult` + ## object available after closing. + ## + ## If you want incremental input, use `setIoCallback()` instead. + ## + ## The `which` parameter controls which streams will be collected; + ## this may include stdin if you want to capture raw input for + ## whatever reason. + ## + ## When `merge` is true, then `stderr` and `stdout` will be treated + ## as one output stream + + ## Currently, this must be called before spawning a process. ctx.setCaptureRaw(which, merge) proc rawMode*(termcap: var Termcap) {.cdecl, importc: "termcap_set_raw_mode", nodecl.} + ## This configures a `Termcap` data structure for `raw` mode, which + ## loosely is non-echoing and unbuffered. There's not quite a firm + ## standard for what raw mode is; but it's a collection of settings, + ## most of which no longer matter in modern terminals. + proc setTimeout*(ctx: var SubProcess, value: var Timeval) {.cdecl, importc: "subproc_set_timeout", nodecl.} + ## This sets how long the process should wait when doing a single + ## poll of file-descriptors to be ready with data to read. If you + ## don't set this, there will be no timeout, and it's possible for + ## the process to die and for the file descriptors associated with + ## them to never return ready. + ## + ## This is *not* an overall timeout for your process, it's a timeout + ## for a single i/o polling cycle. + ## + ## If you have a timeout, a progress callback can be called. + ## + ## Also, when the process is not blocked on the select(), right + ## before the next select we check the status of the subprocess. If + ## it's returned and all its descriptors are marked as closed, and + ## no descriptors that are open are waiting to write, then the + ## subprocess switchboard will exit. + proc clearTimeout*(ctx: var SubProcess) {.cdecl, importc: "subproc_clear_timeout", nodecl.} + ## Remove any set timeout. + proc usePty*(ctx: var SubProcess) {.cdecl, importc: "subproc_use_pty", nodecl.} + ## When this is set on a SubProcess object before the process is + ## spawned, it will cause the process to start using a + ## pseudo-terminal (pty), which, from the point of view of the + ## process being called, simulates a terminal. + ## + ## This can be necessary since some programs only work properly when + ## connected to a terminal, such as `more()` or `less()`. + proc getPtyFd*(ctx: var SubProcess): cint {.cdecl, importc: "subproc_get_pty_fd", nodecl.} + ## When using a PTY, this call returns the file descriptor associated + ## with the child process's terminal. + proc start*(ctx: var SubProcess) {.cdecl, importc: "subproc_start", nodecl.} + ## This starts the sub-process, forking it off. It does NOT poll for + ## input-output. For many apps, you don't need this function; + ## instead, use `run()`. + ## + ## Use this only when you're going to do your own IO polling loop. + proc poll*(ctx: var SubProcess): bool {.cdecl, importc: "subproc_poll", nodecl.} -proc prepareResults*(ctx: var SubProcess) {.cdecl, - importc: "subproc_prepare_results", nodecl.} + ## If you're running your own IO polling loop, this runs the loop + ## one time. You must have previously called `start()`. + ## + ## This returns `true` when called after the process has exited. + proc run*(ctx: var SubProcess) {.cdecl, importc: "subproc_run", nodecl.} + ## This launches a subprocess, and polls it for IO until the process + ## ends, and has no waiting data left to read. + ## + ## Once that happens, you can immediately query results. + ## + ## Note that if you want the subprocess to run in parallel while you + ## do other things, you can either set an IO callback (with + ## `setIoCallback()`) or manually poll by instead using `start()` and + ## then calling `poll()` in your own loop. + proc close*(ctx: var SubProcess) {.cdecl, importc: "subproc_close", nodecl.} + ## Closes down a subprocess; do not call any querying function after + ## this, as the memory will be freed. + proc getPid*(ctx: var SubProcess): Pid {.cdecl, importc: "subproc_get_pid", nodecl.} + ## Returns the process ID associated with the subprocess. This may + ## be called at any point after the process spawns. + proc setExtra*(ctx: var SubProcess, p: pointer) {.cdecl, importc: "subproc_set_extra", nodecl.} + ## This can be used to make arbitrary information available to your + ## I/O callbacks that is specific to the SubProcess instance. + proc getExtra*(ctx: var SubProcess): pointer {.cdecl, importc: "subproc_get_extra", nodecl.} + ## This can be used to retrieve any information set via `setExtra()`. + +proc pausePassthrough*(ctx: var SubProcess, which: SpIoKind) + {.cdecl, importc: "subproc_pause_passthrough", nodecl.} + ## Stops passthrough data from being passed (though pending writes + ## may still succeed). + + +proc resumePassthrough*(ctx: var SubProcess, which: SpIoKind) + {.cdecl, importc: "subproc_resume_passthrough", nodecl.} + ## Resumes passthrough after being paused. For data that didn't get + ## passed during the pause, it will not be seen after the pause + ## either. + ## + ## This allows you to toggle whether input makes it to the + ## subprocess, for instance. + +proc pauseCapture*(ctx: var SubProcess, which: SpIoKind) + {.cdecl, importc: "subproc_pause_capture", nodecl.} + ## Stops capture of a stream. If it's resumed, data published + ## during the pause will NOT be added to the capture. + +proc resumeCapture*(ctx: var SubProcess, which: SpIoKind) + {.cdecl, importc: "subproc_resume_capture", nodecl.} + ## Resumes capturing a stream that's been paused. + proc setIoCallback*(ctx: var SubProcess, which: SpIoKind, callback: SubProcCallback): bool {.cdecl, importc: "subproc_set_io_callback", nodecl, discardable.} + ## Sets up a callback for receiving IO as it is read or written from + ## the terminal. The `which` parameter indicates which streams you + ## wish to subscribe to. You may call this multiple times, for + ## instance, if you'd like to subscribe each stream to a different + ## function, or would like two different functions to receive data. + proc setStartupCallback*(ctx: var SubProcess, callback: SpStartupCallback) {. cdecl, importc: "subproc_set_startup_callback", nodecl .} + ## This allows you to set a callback in the parent process that will + ## run once, after the underlying fork occurs, but before any IO is + ## processed. + proc rawFdWrite*(fd: cint, buf: pointer, l: csize_t) {.cdecl, importc: "write_data", nodecl.} -proc binaryCstringToString*(s: cstring, l: int): string = - for i in 0 ..< l: - result.add(s[i]) + ## An operation that writes from memory to a raw file descriptor. + +template binaryCstringToString*(s: cstring, l: int): string = + ## Don't use this; should probably be rm'd in favor of bytesToString, + ## which it now calls. + bytesToString(s, l) # Nim proxies. Note that the allocCStringArray() calls are going to leak # for the time being. We should clean them up in a destructor. proc initSubProcess*(ctx: var SubProcess, cmd: string, args: openarray[string]) = + ## Initialize a subprocess with the command to call. This does *NOT* + ## run the sub-process. Instead, you can first configure it, and + ## then call `run()` when ready. var cargs = allocCstringArray(args) subproc_init(ctx, cstring(cmd), cargs) proc setEnv*(ctx: var SubProcess, env: openarray[string]) = + ## Explicitly set the environment the subprocess should inherit. If + ## not called before the process is launched, the parent's + ## environment will be inherited. var envp = allocCstringArray(env) ctx.subproc_set_envp(envp) proc pipeToStdin*(ctx: var SubProcess, s: string, close_fd: bool): bool = + ## This allows you to pass input to the subprocess through its + ## stdin. If called before the process spawns, the input will be + ## passed before any other input to the subprocessed process is + ## handled. + ## + ## You can still use this at any point as long as the process's stdin + ## remains open. However, if you pass `true` to the parameter `close_fd`, + ## then the child's stdin will get closed automatically once the + ## write completes. return ctx.subproc_pass_to_stdin(cstring(s), csize_t(s.len()), close_fd) proc getTaggedValue*(ctx: var SubProcess, tag: cstring): string = + ## Lower-level interface to retrieving captured streams. Use + ## `getStd*()` instead. var outlen: csize_t s: cstring @@ -150,21 +309,35 @@ proc getTaggedValue*(ctx: var SubProcess, tag: cstring): string = result = binaryCstringToString(s, int(outlen)) proc getStdin*(ctx: var SubProcess): string = + ## Retrieve stdin, if it was captured. Must be called after the + ## process has completed. ctx.getTaggedValue("stdin") proc getStdout*(ctx: var SubProcess): string = + ## Retrieve stdout, if it was captured. Must be called after the + ## process has completed. + ## + ## If you specified combining stdout and stderr, it will be + ## available here. ctx.getTaggedValue("stdout") proc getStderr*(ctx: var SubProcess): string = + ## Retrieve stdout, if it was captured. Must be called after the + ## process has completed. ctx.getTaggedValue("stderr") proc getExitCode*(ctx: var SubProcess, waitForExit = true): int = + ## Returns the exit code of the process. return int(subproc_get_exit(ctx, waitForExit)) proc getErrno*(ctx: var SubProcess, waitForExit = true): int = + ## If the child died and we received an error, this will contain + ## the value of `errno`. return int(subproc_get_errno(ctx, waitForExit)) proc getSignal*(ctx: var SubProcess, waitForExit = true): int = + ## If the process died as the result of being passed a signal, + ## this will contain the signal number. return int(subproc_get_signal(ctx, waitForExit)) type ExecOutput* = ref object @@ -180,13 +353,43 @@ proc runCommand*(exe: string, closeStdIn = false, pty = false, passthrough = SpIoNone, - passStderrToStdin = false, + passStderrToStdout = false, capture = SpIoOutErr, combineCapture = false, timeoutUsec = 1000, env: openarray[string] = [], waitForExit = true): ExecOutput = - ## One-shot interface + ## This is wrapper that provides a a single call alternative to the + ## builder-style interface. It encompases most of the functionality, + ## though it currently doesn't support setting callbacks. + ## + ## Parameters are: + ## - `exe`: The path to the executable to run. + ## - `args`: The arguments to pass. DO NOT include `exe` again as + ## the first argument, as it is automatically added. + ## - `newStdIn`: If not empty, the contents will be fed to the subprocess + ## after it starts. + ## - `closeStdIn`: If true, will close stdin after writing the contents + ## of `newStdIn` to the subprocess. + ## - `pty`: Whether to use a pseudo-terminal (pty) to run the sub-process. + ## - `passthrough`: Whether to proxy between the parent's stdin/stdout/stderr + ## and the child's. You can specify which ones to proxy. + ## - `passStderrToStdout`: When this is true, the child's stderr is passed + ## to stdout, not stderr. + ## - `capture`: Specifies which file descritors to capture. Captures are + ## available after the process ends. + ## - `combineCapture`: If true, and if you requested capturing both stdout + ## and stderr, will combine them into one stream. + ## - `timeoutUsec`: The number of milliseconds to wait per polling cycle + ## for input. If this is ever exceeded, the subprocess + ## will abort. Set to 0 for unlimited (default is 1000, + ## or 1 second). + ## - `env`: The environment to pass to the subprocess. The default + ## is to inherit the parent's environment. + ## - `waitForExit`: If false, runCommand returns as soon as the subprocess's + ## file descriptors are closed, and doesn't wait for the + ## subprocess to finish. In this case, process exit status + ## will not be reliable. var subproc: SubProcess timeout: Timeval @@ -209,7 +412,7 @@ proc runCommand*(exe: string, if pty: subproc.usePty() if passthrough != SpIoNone: - subproc.setPassthrough(passthrough, passStderrToStdin) + subproc.setPassthrough(passthrough, passStderrToStdout) if capture != SpIoNone: subproc.setCapture(capture, combineCapture) @@ -224,29 +427,48 @@ proc runCommand*(exe: string, result.stdout = subproc.getStdout() result.stderr = subproc.getStderr() -template getStdout*(o: ExecOutput): string = o.stdout -template getStderr*(o: ExecOutput): string = o.stderr -template getExit*(o: ExecOutput): int = o.exitCode +template getStdin*(o: ExecOutput): string = + ## Returns any data captured from the child's stdin stream. + o.stdin +template getStdout*(o: ExecOutput): string = + ## Returns any data captured from the child's stdout stream. + o.stdout +template getStderr*(o: ExecOutput): string = + ## Returns any data captured from the child's stderr stream. + o.stderr +template getExit*(o: ExecOutput): int = + ## Returns any data captured, as passed to the child's std input + o.exitCode +template getPid*(o: ExecOutput): int = + ## Returns the PID from the exited process + o.pid template runInteractiveCmd*(path: string, args: seq[string], passToChild = "", closeFdAfterPass = true, ensureExit = true) = + ## A wrapper for `runCommand` that uses a pseudo-terminal, and lets the + ## user interact with the subcommand. discard runCommand(path, args, passthrough = SpIoAll, capture = SpIoNone, newStdin = passToChild, closeStdin = closeFdAfterPass, pty = true, waitForExit = ensureExit) proc runCmdGetEverything*(exe: string, - args: seq[string], - newStdIn = "", - closeStdIn = true, - passthrough = false, - timeoutUsec = 1000000, - ensureExit = true): ExecOutput = + args: seq[string], + newStdIn = "", + closeStdIn = true, + passthrough = false, + timeoutUsec = 1000000, + ensureExit = true): ExecOutput = + ## A wrapper for `runCommand` that captures all output from the + ## process. This is similar to Nim's `execCmdEx` but allows for + ## optional passthrough, timeouts, and sending an input string to + ## stdin. return runCommand(exe, args, newStdin, closeStdin, pty = false, passthrough = if passthrough: SpIoAll else: SpIoNone, - timeoutUSec = timeoutUsec, capture = SpIoOutErr, waitForExit = ensureExit) + timeoutUSec = timeoutUsec, capture = SpIoOutErr, + waitForExit = ensureExit) proc runPager*(s: string) = var @@ -276,13 +498,5 @@ proc runPager*(s: string) = template runCmdGetOutput*(exe: string, args: seq[string]): string = + ## A legacy wrapper that remains for compatability. execProcess(exe, args = args, options = {}) - -when isMainModule: - var res = runCommand("/bin/cat", @["aes.nim"], pty = true, capture = SpIoAll, - passthrough = SpIoNone) - - echo "pid = ", res.pid - - sleep(2000) - echo res.stdout diff --git a/nimutils/switchboard.nim b/nimutils/switchboard.nim index 3af72ce..2588014 100644 --- a/nimutils/switchboard.nim +++ b/nimutils/switchboard.nim @@ -1,67 +1,372 @@ +## Our switchboard API is focused on being able to arbitrarily +## multiplex IO across file descriptors, and do so using a single +## thread. for simplicitly. Currently, the intent is to be able to +## handle 'normal' applications that often benefit from avoiding the +## complexity of threads. +## +## This also means we use `select()` under the hood, with no option +## yet for `epoll()` or `kqueue()`. So if you've got a very large +## number of connections to multiplex, this isn't the right +## interface. However, this is very well suited for lower volume +## servers and subprocess handling. +## +## When we deal with underlying file decscriptors, we make no +## assumptions about whether they are set to blocking or not. We +## always make sure there's data ready to read, or that we can write +## at least `PIPE_BUF` bytes before we do any operation. +## +## And when multiplexing a listening socket, we only ever accept one +## listener at a time as well. +## +## The end result is that we should never block, and don't care +## whether the file descriptors are set to blocking or not. +## +## We use a pub/sub model, and in each poll, we will always select on +## any open fds that have subscribers available. For writers, we will +## only select on their fds if there are messages waiting to be +## written to that fd, so that we never wake up with nothing to do. +## +## In part, this is handled by having a message queue attached to +## readers. So in the first possible select cycle, (assuming there's +## no string being routed into a file descriptor), we will only +## select() for read fds. +## +## There's no write delay though, as when we re-enter the select loop, +## we'd expect the fds for write to all be ready for data, so th +## select() call will return immediately. +## +## +## Note that this module is a wrapping of the lower-level (C) API. +## Currently, this is primarily meant to be exposed through the +## `subprocess` interface, though we will, in the not-too-distant +## future add a more polished interface to this module that would be +## appropriate for server setups, etc. import os, posix {.pragma: sb, cdecl, importc, nodecl.} static: - {.compile: joinPath(splitPath(currentSourcePath()).head, "switchboard.c").} + {.compile: joinPath(splitPath(currentSourcePath()).head, "c/switchboard.c").} type - SwitchBoard* {.importc: "switchboard_t", header: joinPath(splitPath(currentSourcePath()).head, "switchboard.h") .} = object - Party* {.importc: "party_t", header: joinPath(splitPath(currentSourcePath()).head, "switchboard.h") .} = object + SwitchBoard* {.importc: "switchboard_t", header: "switchboard.h" .} = object + Party* {.importc: "party_t", header: "switchboard.h" .} = object SBCallback* = - proc (i0: pointer, i1: pointer, i2: cstring, i3: int) {. cdecl, gcsafe .} - SBResultObj* {. importc: "sb_result_t", header: joinPath(splitPath(currentSourcePath()).head, "switchboard.h") .} = object + proc (i0: var RootRef, i1: var RootRef, i2: cstring, i3: int) {. cdecl, + gcsafe .} + AcceptCallback* = + proc (i0: var SwitchBoard, fd: cint, addressp: pointer, + addrlenp: pointer) {. cdecl, gcsafe .} + SBCaptures* {. importc: "sb_result_t", header: "switchboard.h" .} = object SbFdPerms* = enum sbRead = 0, sbWrite = 1, sbAll = 2 proc sb_init*(ctx: var SwitchBoard, heap_elems: csize_t) {.sb.} -proc sb_init_party_fd*(ctx: var Switchboard, party: var Party, fd: cint, - perms: SbFdPerms, stopWhenClosed: bool, - closeOnDestroy: bool) {.sb.} -proc sb_init_party_callback*(ctx: var Switchboard, party: var Party, - callback: SBCallback) {.sb.} -proc sb_destroy(ctx: var Switchboard, free: bool) {.sb.} + ## Low-level interface. Use initSwitchboard(). -template initSwitchboard*(ctx: var SwitchBoard, heap_elems: int = 16) = - sb_init(ctx, csize_t(heap_elems)) +proc sb_init_party_fd(ctx: var Switchboard, party: var Party, fd: cint, + perms: SbFdPerms, stopWhenClosed: bool, + closeOnDestroy: bool) {.sb.} + +proc initPartyCallback*(ctx: var Switchboard, party: var Party, + callback: SBCallback) {.cdecl, + importc: "sb_init_party_callback", nodecl .} + ## This sets up a callback to receive incremental data that + ## has been read from any file descriptor, except listening sockets. + ## + ## Any state information can be passed to this callback via the + ## as-yet-unwrapped `sb_set_extra()` and retrieved by the similarly + ## unwrapped `sb_get_party_extra()`. + +proc sb_init_party_listener(ctx: var Switchboard, party: var Party, + sockfd: int, callback: AcceptCallback, + stopWhenClosed: bool, closeOnDestroy: bool) {.sb.} + +proc initPartyListener*(ctx: var Switchboard, party: var Party, + sockfd: int, callback: AcceptCallback, + stopWhenClosed = false, closeOnDestroy = true) = + ## This sets up monitoring of a socket that is listening for connections. + ## The provided callback will be called whenever there is a listening + ## socket waiting to be read. + ctx.sb_init_party_listener(party, sockfd, callback, stopWhenClosed, + closeOnDestroy) + +proc sb_init_party_input_buf(ctx: var Switchboard, party: var Party, + input: cstring, l: csize_t, dup: bool, + free: bool, close_fd_when_done: bool) {.sb.} + +proc sb_init_party_output_buf(ctx: var Switchboard, party: var Party, + tag: cstring, l: csize_t) {.sb.} + +proc initPartyCapture*(ctx: var Switchboard, party: var Party, + prealloc = 4096, tag: static[string]) = + ## Sets up a capture buffer that you can send output to, as long as + ## the source you're routing from is a non-string writer. + ## + ## The underlying api assumes that it never has to free the passed + ## tag and that it will always exit, so in this variant, the tag + ## must point to static memory. + ctx.sb_init_party_output_buf(party, tag, csize_t(prealloc)) + +proc unsafeInitPartyCapture*(ctx: var Switchboard, party: var Party, + prealloc = 4096, tag: string) = + ## This is the same as initPartyCapture, except you use a string + ## that you assert will not be freed before the switchboard has been + ## torn down. + ## + ## If you don't follow this contract, you will probably either + ## crash, or get garbage. + ## + ## We may refactor the underlying implementation to address this, + ## but don't count on it! + ctx.sb_init_party_output_buf(party, tag, csize_t(prealloc)) + +proc sb_monitor_pid(ctx: var Switchboard, pid: Pid, stdin: ptr Party, + stdout: ptr Party, stderr: ptr Party, shutdown: bool) {.sb.} -template initPartyFd*(ctx: var SwitchBoard, party: var Party, fd: int, - perms: SbFdPerms, stopWhenClosed = false, - closeOnDestroy = false) = +proc monitorProcess*(ctx: var Switchboard, pid: Pid, stdin: ref Party = nil, + stdout: ref Party = nil, stderr: ref Party = nil, + shutdown: bool = false) = + ## Sets up a process monitor for a switchboard. Currently there is + ## no direct API for retrieving data from the monitors. We need to + ## fix this. If you're using the subprocess interface, you're + ## indirectly getting access to a single monitor. + ## + ## If you attach parties to the process that present the stdin, + ## stdout or stderr of the process, the system can detect when they + ## close. This is most useful if you want to exit when the process + ## disappears, which we might notice if trying to write to its file + ## descriptor, but it fails. + ## + ## If that happens and `shutdown` is true, then the remaining data + ## is drained from the read side of this file descriptor, all queued + ## writes are completed, then the switchboard will exit (possibly + ## with active file descriptors). A few reads from other file + ## descriptors could get services while waiting for the shutdown. + ctx.sb_monitor_pid(pid, cast[ptr Party](stdin), cast[ptr Party](stdout), + cast[ptr Party](stderr), shutdown) + +proc initPartyStrInput*(ctx: var Switchboard, party: var Party, + input: string = "", closeFdWhenDone: bool = false) = + ## Initializes an input string that can be the source when routing + ## to a file descriptor. This is generally useful for sending data + ## to sockets, or to subprocesses. + ## + ## This object can be re-used as much as you like by calling + ## `party.setString()`. + ## + ## However, neither this function nor `setString()` pass data + ## explicitly; with strings, you must call `route` every time to any + ## party you want to send the string to. We may add a wrapper object + ## that automates this process. + ## + ## If `closeFdWhenDone` is set, then this is intended to be a one-off, + ## and any file descriptors this is scheduled to write to will be + ## closed automatically once the write is completed. This allows you + ## to close the stdin of a subprocess after the string gets written. + ctx.sb_init_party_input_buf(party, cstring(input), csize_t(input.len()), + true, true, close_fd_when_done) + +proc sb_party_input_buf_new_string(party: var Party, input: cstring, + l: csize_t, dup: bool, + free: bool, closeFd: bool) {.sb.} + +proc setString*(party: var Party, input: string, closeAfter: bool = false) = + ## If a party is a string input buffer, this will update the string + ## it sends out when you call `route` on it. + ## + ## If this doesn't represent an string input party, nothing will happen. + ## + ## If `closeAfter` is true, then once this string is written, any + ## subscriber will be closed. + party.sb_party_input_buf_new_string(cstring(input), csize_t(input.len()), + true, true, closeAfter) +proc initPartyFd*(ctx: var SwitchBoard, party: var Party, fd: int, + perms: SbFdPerms, stopWhenClosed = false, + closeOnDestroy = false) = + ## Initializes a file descriptor that can be both a subscriber and + ## subscribed to, depending on the fd's permissions (which are + ## passed, not discovered). + ## + ## The `perms` field should be sbRead for O_RDONLY, sbWrite for O_WRONLY + ## or sbAll for O_RDWR + ## + ## The `stopWhenClosed` field closes down the switchboard when I/O + ## to this fd fails. + ## + ## If `closeOnDestroy` is true, we will call close() on the fd for + ## you whenever the switchboard is torn down. sb_init_party_fd(ctx, party, cint(fd), perms, stopWhenClosed, closeOnDestroy) -template initPartyCallback*(ctx: var SwitchBoard, party: var Party, - cb: SBCallback) = - sb_init_party_callback(ctx, party, cb); +proc sb_destroy(ctx: var Switchboard, free: bool) {.sb.} + +template initSwitchboard*(ctx: var SwitchBoard, heap_elems: int = 16) = + sb_init(ctx, csize_t(heap_elems)) + ## Initialize a switchboard object. proc route*(ctx: var Switchboard, src: var Party, dst: var Party): bool {.cdecl, importc: "sb_route", nodecl, discardable.} + ## Route messages from the `src` object to the `dst` object. + ## Basically, the `dst` party subscribes to messages from the `src`. + ## These subscriptions shouldn't be removed, but can be paused and + ## resumed (pausing and never resuming is tantamount to removing). + +proc pauseRoute*(ctx: var Switchboard, src: var Party, dst: var Party): bool + {.cdecl, importc: "sb_pause_route", nodecl, discardable.} + ## Akin to removing a route subscription, except that you can easily + ## re-subscribe if you wish by calling `resumeRoute` + +proc resumeRoute*(ctx: var Switchboard, src: var Party, dst: var Party): bool + {.cdecl, importc: "sb_resume_route", nodecl, discardable.} + ## Restarts a previous route / subscription that has been paused. + +proc routeIsActive*(ctx: var Switchboard, src: var Party, dst: var Party): bool + {.cdecl, importc: "sb_route_is_active", nodecl, discardable.} + ## Returns true if the subscription is active, meaning it exists, + ## neither side is closed, and the subscription is not paused. + +proc routeIsPaused*(ctx: var Switchboard, src: var Party, dst: var Party): bool + {.cdecl, importc: "sb_route_is_paused", nodecl, discardable.} + ## Returns true if the subscription is active but paused, meaning a + ## subscribed happened, but it was paused. If either side is closed, + ## this will return `false`, even if it had previously been paused. + +proc routeIsSubscribed*(ctx: var Switchboard, src: var Party, + dst: var Party): bool + {.cdecl, importc: "sb_route_is_subscribed", nodecl, discardable.} + ## Returns true if the subscription is active, meaning a subscribed + ## happened, and neither side is closed. However, it may be either + ## paused or unpaused. proc setTimeout*(ctx: var Switchboard, value: var Timeval) {.cdecl, importc: "sb_set_io_timeout", nodecl.} + ## Sets the amount of time that one polling loop blocks waiting for + ## I/O. If you're definitely going to wait forever until the + ## switchboard ends, then this can be unlimited (see + ## `clearTimeout()`). + ## + ## Generally, you want this low enough to make sure you're + ## responsive to a hang of some sort, but not TOO low, as it will + ## unnecessarily drive up CPU. proc clearTimeout*(ctx: var Switchboard) {.cdecl, importc: "sb_clear_io_timeout", nodecl.} + ## Removes any polling timeout; if there's no IO on the switchboard, + ## polling will hang until there is. proc operateSwitchboard*(ctx: var Switchboard, toCompletion: bool): bool {.cdecl, importc: "sb_operate_switchboard", nodecl, discardable.} + ## Low-level interface; use run() instead. + +proc sb_set_extra(ctx: var Switchboard, extra: RootRef) {.sb.} +proc sb_set_party_extra(ctx: var Party, extra: RootRef) {.sb.} + +proc getExtraData*(ctx: var Switchboard): RootRef + {.cdecl, importc: "sb_get_extra", nodecl.} + ## Retrieves any extra data stored, specific to a switchboard. + +proc getExtraData*(ctx: var Party): RootRef + {.cdecl, importc: "sb_get_party_extra", nodecl.} + ## Retrieves any extra data stored for the party. + +proc clearExtraData*(ctx: var Switchboard) = + ## Removes any stored extra data (setting it to nil). + let x = ctx.getExtraData() + + if x != nil: + ctx.sb_set_extra(RootRef(nil)) + GC_unref(x) + +proc clearExtraData*(ctx: var Party) = + ## Removes any stored extra data (setting it to nil). + let x = ctx.getExtraData() + + if x != nil: + ctx.sb_set_party_extra(RootRef(nil)) + GC_unref(x) + +proc setExtraData*(ctx: var Switchboard, extra: RootRef) = + ## Sets any extra data that will get passed to IO callbacks that is + ## specific to this switchboard. It must be a "ref" object + ## (inheriting from RootRef). The reference will be held until the + ## process exits, or it is replaced. + ## + ## This call should only ever be called from one thread at a time; + ## there's a race condition otherwise, where you could end up trying + ## to double-free the old object, and end up leaking the object that + ## gets written first. + let x = ctx.getExtraData() + + if x != nil: + GC_unref(x) + + ctx.sb_set_extra(extra) + + if extra != RootRef(nil): + GC_ref(extra) + +proc setExtraData*(ctx: var Party, extra: RootRef) = + ## Sets any extra data that will get passed to IO callbacks that is + ## specific to this party (will be passed back to you in the second + ## parameter of the callback). It must be a "ref" object (inheriting + ## from RootRef). The reference will be held until the process + ## exits, or it is replaced. + ## + ## This call should only ever be called from one thread at a time; + ## there's a race condition otherwise, where you could end up trying + ## to double-free the old object, and end up leaking the object that + ## gets written first. + let x = ctx.getExtraData() + + if x != nil: + GC_unref(x) + + ctx.sb_set_party_extra(extra) + + if extra != RootRef(nil): + GC_ref(extra) + +proc `=destroy`*(ctx: Switchboard) = + var copy = ctx + copy.clearExtraData() + copy.sb_destroy(false) + +proc `=destroy`*(ctx: Party) = + var copy = ctx + copy.clearExtraData() + +proc sb_result_destroy(res: var SBCaptures) {.sb.} + +proc `=destroy*`(res: var SBCaptures) = + res.sb_result_destroy() + +proc sb_result_get_capture(res: var SBCaptures, tag: cstring, + borrow: bool): cstring {.sb.} + +proc getCapture*(res: var SBCaptures, tag: string): string = + ## Returns a specific process capture by tag. + return $(res.sb_result_get_capture(cstring(tag), true)) -template run*(ctx: var Switchboard, toCompletion = false) = - operateSwitchboard(ctx, toCompletion) +proc run*(ctx: var Switchboard, toCompletion = true): bool {.discardable.} = + ## Runs the switchboard IO polling cycle. By default, this will keep + ## running until there are no subscriptions that could possibly be + ## serviced without a new subscription. + ## + ## However, if you set `toCompletion` to `false`, it will run only + ## one polling cycle. In this mode, you're expected to manually + ## poll when it's convenient for your application. + ## + ## Once the switchboard has ended, call `ctx.getResults()` + ## to get any capture or process info. -proc close*(ctx: var Switchboard) = ctx.sb_destroy(false) + if not operateSwitchboard(ctx, toCompletion): + ctx.clearExtraData() # Don't need it anymore, why wait till close? + return true + elif toCompletion: + return true + else: + return false # Not yet wrapped: -## extern void sb_init_party_listener(switchboard_t *, party_t *, int, -## accept_cb_t, bool, bool); -## extern void sb_init_party_input_buf(switchboard_t *, party_t *, char *, -## size_t, bool, bool); -## extern void sb_init_party_output_buf(switchboard_t *, party_t *, char *, -## size_t); ## extern void sb_monitor_pid(switchboard_t *, pid_t, party_t *, party_t *, ## party_t *, bool); -## extern void *sb_get_extra(switchboard_t *); -## extern void sb_set_extra(switchboard_t *, void *); -## extern void *sb_get_party_extra(party_t *); -## extern void sb_set_party_extra(party_t *, void *); -## extern sb_result_t *sb_automatic_switchboard(switchboard_t *, bool); diff --git a/nimutils/texttable.nim b/nimutils/texttable.nim index 6505689..fd8a704 100644 --- a/nimutils/texttable.nim +++ b/nimutils/texttable.nim @@ -1,15 +1,13 @@ ## :Author: John Viega (john@crashoverride.com) ## :Copyright: 2023, Crash Override, Inc. -# Pretty basic table formatting. Works with fixed width unicode, though -# I don't factor out non-printable spaces right now, I just count runes. - -import rope_construct, rope_ansirender, markdown, unicode, std/terminal, - unicodeid +import rope_base, rope_construct, rope_prerender, rope_styles, markdown, + unicode, unicodeid, std/terminal proc formatCellsAsMarkdownList*(base: seq[seq[string]], toEmph: openarray[string], firstCellPrefix = "\n## "): string = + ## Deprecated; old code. Kept for short term due to compatability. for row in base: result &= "\n" if firstCellPrefix != "": @@ -34,7 +32,7 @@ proc formatCellsAsHtmlTable*(base: seq[seq[string]], headers: openarray[string] = [], mToHtml = true, verticalHeaders = false): string = - + ## Deprecated; old code. Kept for short term due to compatability. if len(base) == 0: raise newException(ValueError, "Table is empty.") @@ -79,6 +77,7 @@ proc filterEmptyColumns*(inrows: seq[seq[string]], headings: openarray[string], emptyVals = ["", "None", "None", "[]"]): (seq[seq[string]], seq[string]) = + ## Deprecated. From the pre-rope days. var columnHasValues: seq[bool] returnedHeaders: seq[string] @@ -107,19 +106,29 @@ proc filterEmptyColumns*(inrows: seq[seq[string]], return (newRows, returnedHeaders) -proc instantTable*(cells: seq[string], html = false): string = +proc instantTable*[T: string|Rope](cells: openarray[T], tableCaption = Rope(nil), + width = -1, borders = BorderAll, + boxStyle = BoxStyleDouble): Rope = + ## Given a flat list of items to format into a table, figures out how + ## many equal-sized columns fit cleanly into the available width, given + ## the text, and formats it all into a table. var - remainingWidth = terminalWidth() + remainingWidth: int numcol = 0 maxWidth = 0 - row: seq[string] = @[] - rows: seq[seq[string]] - + row: seq[Rope] = @[] + rows: seq[Rope] = @[] # This gives every column equal width, and assumes space for borders # and pad. + if width <= 0: + remainingWidth = terminalWidth() + width + + else: + remainingWidth = width + for item in cells: - let w = item.strip().runeLength() + let w = item.runeLength() if w > maxWidth: maxWidth = w @@ -129,25 +138,104 @@ proc instantTable*(cells: seq[string], html = false): string = for i, item in cells: if i != 0 and i mod numcol == 0: - rows.add(row) + rows.add(tr(row)) row = @[] - row.add(item.strip()) + row.add(td(item)) var n = len(cells) while n mod numcol != 0: - row.add("") + row.add(td("")) n = n + 1 - rows.add(row) + rows.add(tr(row)) + + result = table(tbody(rows), caption = tableCaption) + result = result.setBorders(borders).boxStyle(boxStyle) + result = result.setWidth(remainingWidth) + +template instantTableNoHeaders[T: string|Rope](cells: seq[seq[T]], + tableCaption: Rope): Rope = + var + row: seq[Rope] = @[] + rows: seq[Rope] = @[] + + for cellrow in cells: + for item in cellrow: + row.add(td(item)) + rows.add(tr(row)) + row = @[] + + colors(table(tbody(rows), thead(@[]), caption = tableCaption)) + +template instantTableHorizontalHeaders[T: string|Rope](cells: seq[seq[T]], + tableCaption: Rope): Rope = + var + row: seq[Rope] = @[] + rows: seq[Rope] = @[] + + for i, cellrow in cells: + if i == 0: + for item in cellrow: + row.add(th(item)) + else: + for item in cellrow: + row.add(td(item)) + rows.add(tr(row)) + row = @[] + + table(tbody(rows), caption = tableCaption) + +template instantTableVerticalHeaders[T: string|Rope](cells: seq[seq[T]], + tableCaption: Rope): Rope = + var + row: seq[Rope] = @[] + rows: seq[Rope] = @[] + + for cellrow in cells: + for i, item in cellrow: + if i == 0: + row.add(th(item)) + else: + row.add(td(item)) + rows.add(tr(row)) + row = @[] + + table(tbody(rows), caption = tableCaption) + +proc quickTable*[T: string|Rope](cells: seq[seq[T]], verticalHeaders = false, + noheaders = false, caption = Rope(nil), width = -1, + borders = BorderAll, boxStyle = BoxStyleDouble, + colPcts: seq[int] = @[]): Rope = + if cells.len() == 0: + raise newException(ValueError, "No cells passed") + + if noHeaders: + result = colors(cells.instantTableNoHeaders(caption)) + elif not verticalHeaders: + result = colors(cells.instantTableHorizontalHeaders(caption)) + else: + result = colors(cells.instantTableVerticalHeaders(caption)) + + result.setBorders(borders).boxStyle(boxStyle) + + if cells.len() == 1 and cells[0].len() == 1: + # Special treatment for callouts. + result.setClass("callout", recurse = true) + + if colPcts.len() != 0: + result.colPcts(colPcts) + + if width >= 0: + result = result.setWidth(width) - result = rows.formatCellsAsHtmlTable() - if not html: - result = result.stylizeHtml() +template instantTableWithHeaders*(cells: seq[seq[string]], horizontal = true, + caption = Rope(nil)): string = + ## Deprecated, for compatability with older con4m. + $(instantTable(cells, horizontal, true, caption)) -proc instantTableWithHeaders*(cells: seq[seq[string]]): string = - let - headers = cells[0] - rest = cells[1 .. ^1] - return rest.formatCellsAsHtmlTable(headers).stylizeHtml() +proc callOut*[T: string | Rope](contents: T, width = -1, borders = BorderAll, + boxStyle = BoxStyleDouble): Rope = + result = quickTable(@[@[contents.center()]], false, false, Rope(nil), + width, borders, boxStyle)