From ed598d6b6af65c387cb51b6d4ed1e9cfa16a1f73 Mon Sep 17 00:00:00 2001 From: megh Date: Thu, 7 Dec 2023 13:48:52 -0700 Subject: [PATCH 1/9] adds basic tooling --- .github/workflows/release.yml | 121 ++++++ .gitignore | 4 - .vscode/settings.json | 2 +- CHANGELOG.md | 5 + Cargo.lock | 779 ++++++++++++++++++++++++++++++++++ Cargo.toml | 17 +- README.md | 38 +- src/lib.rs | 8 +- src/main.rs | 47 +- src/migration/entry.rs | 177 ++++++++ src/migration/example.rs | 14 + src/migration/util.rs | 68 +++ 12 files changed, 1241 insertions(+), 39 deletions(-) create mode 100644 .github/workflows/release.yml create mode 100644 CHANGELOG.md create mode 100644 Cargo.lock create mode 100644 src/migration/entry.rs create mode 100644 src/migration/example.rs create mode 100644 src/migration/util.rs diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..d65f71b --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,121 @@ +name: release + +on: + push: + tags: + - '*-?v[0-9]+*' + +jobs: + metadata: + runs-on: ubuntu-latest + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + outputs: + version: ${{ steps.generate.outputs.version }} + steps: + - uses: actions/checkout@v4 + - uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: stable + - uses: Swatinem/rust-cache@v2 + with: + shared-key: "builds" + - uses: actions-rs/install@v0.1 + with: + crate: parse-changelog + version: latest + - name: generate + id: generate + # multi-line strings are special. + # see: https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#multiline-strings + run: | + VERSION=$(cargo metadata | jq -r '.packages[] | select(.name == "reachability-toolkit") | .version') + TITLE=$(parse-changelog CHANGELOG.md --title) + echo "version=$VERSION" >> "$GITHUB_OUTPUT" + + parse-changelog CHANGELOG.md $VERSION >> release_changelog.md + gh release create ${{ github.ref_name }} --draft --title="$TITLE" --notes-file=release_changelog.md + gh release upload ${{ github.ref_name }} LICENSE CHANGELOG.md README.md --clobber + + build: + strategy: + matrix: + os: ['windows-latest', 'ubuntu-latest', 'macos-latest'] + include: + - os: ubuntu-latest + os-name: linux + toolchain: stable + - os: macos-latest + os-name: macos + toolchain: stable + - os: windows-latest + os-name: windows + toolchain: stable + + needs: [metadata] + if: ${{ needs.metadata.result == 'success' }} + name: ${{ matrix.os-name }}-build + runs-on: ${{ matrix.os }} + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + RELEASE_VERSION: ${{ needs.metadata.outputs.version }} + steps: + - uses: actions/checkout@v4 + - uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: ${{ matrix.toolchain }} + - uses: Swatinem/rust-cache@v2 + with: + shared-key: "builds" + - uses: actions-rs/install@v0.1 + if: ${{ matrix.os-name == 'linux' }} + with: + crate: cross + version: latest + - uses: actions-rs/install@v0.1 + with: + crate: cargo-nextest + version: latest + # Each platform has its own particulars, can't do cool matrix things here. + # Just fall back to conditionals. + - name: "build and upload for linux" + if: ${{ matrix.os-name == 'linux' }} + run: | + mkdir release + cross build --features jemalloc --target=x86_64-unknown-linux-musl --release + mv target/x86_64-unknown-linux-musl/release/broker release/broker-$RELEASE_VERSION-x86_64-linux + chmod +x release/* + gh release upload ${{ github.ref_name }} $(find release -mindepth 1 | xargs) --clobber + + - name: "build and upload for macos" + if: ${{ matrix.os-name == 'macos' }} + run: | + mkdir release + rustup target add aarch64-apple-darwin + rustup target add x86_64-apple-darwin + cargo build --target=aarch64-apple-darwin --release + cargo build --target=x86_64-apple-darwin --release + mv target/aarch64-apple-darwin/release/broker release/broker-$RELEASE_VERSION-aarch64-macos + mv target/x86_64-apple-darwin/release/broker release/broker-$RELEASE_VERSION-x86_64-macos + chmod +x release/* + gh release upload ${{ github.ref_name }} $(find release -mindepth 1 | xargs) --clobber + + - name: "build and upload for windows" + if: ${{ matrix.os-name == 'windows' }} + run: | + mkdir release + cargo build --release + mv target/release/broker.exe release/broker-$env:RELEASE_VERSION-x86_64-windows.exe + gh release upload ${{ github.ref_name }} release/broker-$env:RELEASE_VERSION-x86_64-windows.exe --clobber + + publish-release: + needs: [metadata, build] + if: ${{ needs.metadata.result == 'success' && needs.build.result == 'success' }} + runs-on: ubuntu-latest + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + steps: + - uses: actions/checkout@v4 + - run: gh release edit ${{ github.ref_name }} --draft=false \ No newline at end of file diff --git a/.gitignore b/.gitignore index 196e176..a3040a7 100644 --- a/.gitignore +++ b/.gitignore @@ -3,10 +3,6 @@ debug/ target/ -# Remove Cargo.lock from gitignore if creating an executable, leave it for libraries -# More information here https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html -Cargo.lock - # These are backup files generated by rustfmt **/*.rs.bk diff --git a/.vscode/settings.json b/.vscode/settings.json index 6d0dc2f..4006a6e 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -13,5 +13,5 @@ "rustup", "Swatinem", "taiki" - ] + ], } diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..ef82e1a --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,5 @@ +## v0.0.1 + +Features: +- Adds `example` command, to create `example.yml` file. +- Adds `migrate` command, to create `YYYYMMDDHHMMSS-vulnComponent-automated.js` migration file. \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 0000000..0d19ab5 --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,779 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "aho-corasick" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2969dcb958b36655471fc61f7e416fa76033bdd4bfed0678d8fee1e2d07a1f0" +dependencies = [ + "memchr", +] + +[[package]] +name = "android-tzdata" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + +[[package]] +name = "anstream" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ab91ebe16eb252986481c5b62f6098f3b698a45e34b5b98200cf20dd2484a44" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7079075b41f533b8c61d2a4d073c4676e1f8b249ff94a393b0595db304e0dd87" + +[[package]] +name = "anstyle-parse" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c75ac65da39e5fe5ab759307499ddad880d724eed2f6ce5b5e8a26f4f387928c" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3a318f1f38d2418400f8209655bfd825785afd25aa30bb7ba6cc792e4596748" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1cd54b81ec8d6180e24654d0b371ad22fc3dd083b6ff8ba325b72e00c87660a7" +dependencies = [ + "anstyle", + "windows-sys", +] + +[[package]] +name = "anyhow" +version = "1.0.75" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4668cab20f66d8d020e1fbc0ebe47217433c1b6c8f2040faf858554e394ace6" + +[[package]] +name = "autocfg" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" + +[[package]] +name = "bumpalo" +version = "3.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f30e7476521f6f8af1a1c4c0b8cc94f0bee37d91763d0ca2665f299b6cd8aec" + +[[package]] +name = "cc" +version = "1.0.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0" +dependencies = [ + "libc", +] + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "chrono" +version = "0.4.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f2c685bad3eb3d45a01354cedb7d5faa66194d1d58ba6e267a8de788f79db38" +dependencies = [ + "android-tzdata", + "iana-time-zone", + "js-sys", + "num-traits", + "wasm-bindgen", + "windows-targets 0.48.5", +] + +[[package]] +name = "clap" +version = "4.4.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfaff671f6b22ca62406885ece523383b9b64022e341e53e009a62ebc47a45f2" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.4.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a216b506622bb1d316cd51328dce24e07bdff4a6128a47c7e7fad11878d5adbb" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf9804afaaf59a91e75b022a30fb7229a7901f60c755489cc61c9b423b836442" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.39", +] + +[[package]] +name = "clap_lex" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "702fc72eb24e5a1e48ce58027a675bc24edd52096d5397d4aea7c6dd9eca0bd1" + +[[package]] +name = "colorchoice" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" + +[[package]] +name = "core-foundation-sys" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" + +[[package]] +name = "equivalent" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" + +[[package]] +name = "getset" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e45727250e75cc04ff2846a66397da8ef2b3db8e40e0cef4df67950a07621eb9" +dependencies = [ + "proc-macro-error", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "hashbrown" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604" + +[[package]] +name = "heck" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" + +[[package]] +name = "iana-time-zone" +version = "0.1.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8326b86b6cff230b97d0d312a6c40a60726df3332e721f72a1b035f451663b20" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "indexmap" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d530e1a18b1cb4c484e6e34556a0d948706958449fca0cab753d649f2bce3d1f" +dependencies = [ + "equivalent", + "hashbrown", +] + +[[package]] +name = "indoc" +version = "2.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e186cfbae8084e513daff4240b4797e342f988cecda4fb6c939150f96315fd8" + +[[package]] +name = "itoa" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" + +[[package]] +name = "js-sys" +version = "0.3.66" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cee9c64da59eae3b50095c18d3e74f8b73c0b86d2792824ff01bbce68ba229ca" +dependencies = [ + "wasm-bindgen", +] + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + +[[package]] +name = "libc" +version = "0.2.150" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89d92a4743f9a61002fae18374ed11e7973f530cb3a3255fb354818118b2203c" + +[[package]] +name = "log" +version = "0.4.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" + +[[package]] +name = "memchr" +version = "2.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f665ee40bc4a3c5590afb1e9677db74a508659dfd71e126420da8274909a0167" + +[[package]] +name = "num-traits" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39e3200413f237f41ab11ad6d161bc7239c84dcb631773ccd7de3dfe4b5c267c" +dependencies = [ + "autocfg", +] + +[[package]] +name = "once_cell" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" + +[[package]] +name = "proc-macro-error" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" +dependencies = [ + "proc-macro-error-attr", + "proc-macro2", + "quote", + "syn 1.0.109", + "version_check", +] + +[[package]] +name = "proc-macro-error-attr" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" +dependencies = [ + "proc-macro2", + "quote", + "version_check", +] + +[[package]] +name = "proc-macro2" +version = "1.0.70" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39278fbbf5fb4f646ce651690877f89d1c5811a3d4acb27700c1cb3cdb78fd3b" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "reachability-toolkit" +version = "0.1.0" +dependencies = [ + "anyhow", + "chrono", + "clap", + "indoc", + "lazy_static", + "regex", + "serde", + "serde_json", + "serde_yaml", + "srclib", + "test-case", +] + +[[package]] +name = "regex" +version = "1.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "380b951a9c5e80ddfd6136919eef32310721aa4aacd4889a8d39124b026ab343" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f804c7828047e88b2d32e2d7fe5a105da8ee3264f01902f796c8e067dc2483f" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f" + +[[package]] +name = "rustversion" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ffc183a10b4478d04cbbbfc96d0873219d962dd5accaff2ffbd4ceb7df837f4" + +[[package]] +name = "ryu" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741" + +[[package]] +name = "serde" +version = "1.0.193" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25dd9975e68d0cb5aa1120c288333fc98731bd1dd12f561e468ea4728c042b89" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.193" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43576ca501357b9b071ac53cdc7da8ef0cbd9493d8df094cd821777ea6e894d3" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.39", +] + +[[package]] +name = "serde_json" +version = "1.0.108" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d1c7e3eac408d115102c4c24ad393e0821bb3a5df4d506a80f85f7a742a526b" +dependencies = [ + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "serde_yaml" +version = "0.9.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3cc7a1570e38322cfe4154732e5110f887ea57e22b76f4bfd32b5bdd3368666c" +dependencies = [ + "indexmap", + "itoa", + "ryu", + "serde", + "unsafe-libyaml", +] + +[[package]] +name = "srclib" +version = "0.1.0" +source = "git+https://github.com/fossas/foundation-libs#4bc3762e73f371717566fb075d02e1d25b21146e" +dependencies = [ + "getset", + "lazy_static", + "regex", + "serde", + "strum", + "thiserror", + "typed-builder", +] + +[[package]] +name = "strsim" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" + +[[package]] +name = "strum" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "063e6045c0e62079840579a7e47a355ae92f60eb74daaf156fb1e84ba164e63f" +dependencies = [ + "strum_macros", +] + +[[package]] +name = "strum_macros" +version = "0.24.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e385be0d24f186b4ce2f9982191e7101bb737312ad61c1f2f984f34bcf85d59" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "rustversion", + "syn 1.0.109", +] + +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "2.0.39" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23e78b90f2fcf45d3e842032ce32e3f2d1545ba6636271dcbf24fa306d87be7a" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "test-case" +version = "3.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb2550dd13afcd286853192af8601920d959b14c401fcece38071d53bf0768a8" +dependencies = [ + "test-case-macros", +] + +[[package]] +name = "test-case-core" +version = "3.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adcb7fd841cd518e279be3d5a3eb0636409487998a4aff22f3de87b81e88384f" +dependencies = [ + "cfg-if", + "proc-macro2", + "quote", + "syn 2.0.39", +] + +[[package]] +name = "test-case-macros" +version = "3.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c89e72a01ed4c579669add59014b9a524d609c0c88c6a585ce37485879f6ffb" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.39", + "test-case-core", +] + +[[package]] +name = "thiserror" +version = "1.0.50" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9a7210f5c9a7156bb50aa36aed4c95afb51df0df00713949448cf9e97d382d2" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.50" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "266b2e40bc00e5a6c09c3584011e08b06f123c00362c92b975ba9843aaaa14b8" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.39", +] + +[[package]] +name = "typed-builder" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89851716b67b937e393b3daa8423e67ddfc4bbbf1654bcf05488e95e0828db0c" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "unicode-ident" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" + +[[package]] +name = "unsafe-libyaml" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f28467d3e1d3c6586d8f25fa243f544f5800fec42d97032474e17222c2b75cfa" + +[[package]] +name = "utf8parse" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" + +[[package]] +name = "version_check" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" + +[[package]] +name = "wasm-bindgen" +version = "0.2.89" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ed0d4f68a3015cc185aff4db9506a015f4b96f95303897bfa23f846db54064e" +dependencies = [ + "cfg-if", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.89" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b56f625e64f3a1084ded111c4d5f477df9f8c92df113852fa5a374dbda78826" +dependencies = [ + "bumpalo", + "log", + "once_cell", + "proc-macro2", + "quote", + "syn 2.0.39", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.89" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0162dbf37223cd2afce98f3d0785506dcb8d266223983e4b5b525859e6e182b2" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.89" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0eb82fcb7930ae6219a7ecfd55b217f5f0893484b7a13022ebb2b2bf20b5283" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.39", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.89" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ab9b36309365056cd639da3134bf87fa8f3d86008abf99e612384a6eecd459f" + +[[package]] +name = "windows-core" +version = "0.51.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1f8cf84f35d2db49a46868f947758c7a1138116f7fac3bc844f43ade1292e64" +dependencies = [ + "windows-targets 0.48.5", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.0", +] + +[[package]] +name = "windows-targets" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" +dependencies = [ + "windows_aarch64_gnullvm 0.48.5", + "windows_aarch64_msvc 0.48.5", + "windows_i686_gnu 0.48.5", + "windows_i686_msvc 0.48.5", + "windows_x86_64_gnu 0.48.5", + "windows_x86_64_gnullvm 0.48.5", + "windows_x86_64_msvc 0.48.5", +] + +[[package]] +name = "windows-targets" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a18201040b24831fbb9e4eb208f8892e1f50a37feb53cc7ff887feb8f50e7cd" +dependencies = [ + "windows_aarch64_gnullvm 0.52.0", + "windows_aarch64_msvc 0.52.0", + "windows_i686_gnu 0.52.0", + "windows_i686_msvc 0.52.0", + "windows_x86_64_gnu 0.52.0", + "windows_x86_64_gnullvm 0.52.0", + "windows_x86_64_msvc 0.52.0", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7764e35d4db8a7921e09562a0304bf2f93e0a51bfccee0bd0bb0b666b015ea" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbaa0368d4f1d2aaefc55b6fcfee13f41544ddf36801e793edbbfd7d7df075ef" + +[[package]] +name = "windows_i686_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a28637cb1fa3560a16915793afb20081aba2c92ee8af57b4d5f28e4b3e7df313" + +[[package]] +name = "windows_i686_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffe5e8e31046ce6230cc7215707b816e339ff4d4d67c65dffa206fd0f7aa7b9a" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d6fa32db2bc4a2f5abeacf2b69f7992cd09dca97498da74a151a3132c26befd" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a657e1e9d3f514745a572a6846d3c7aa7dbe1658c056ed9c3344c4109a6949e" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dff9641d1cd4be8d1a070daf9e3773c5f67e78b4d9d42263020c057706765c04" diff --git a/Cargo.toml b/Cargo.toml index 6184e61..86a40a3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,8 +1,21 @@ [package] -name = "template-rust" +name = "reachability-toolkit" version = "0.1.0" edition = "2021" +authors = ["github.com/fossas"] # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - [dependencies] +clap = { version = "4.0.26", features = ["derive", "cargo"] } +serde = { version = "1.0.147", features = ["derive"] } +serde_json = "1.0.88" +serde_yaml = "*" +anyhow = "1.0" +regex = "1" +chrono = "0.4" +srclib = { git = "https://github.com/fossas/foundation-libs" } +indoc = "2" +lazy_static="*" + +[dev-dependencies] +test-case = "*" diff --git a/README.md b/README.md index 9cbcc40..6411369 100644 --- a/README.md +++ b/README.md @@ -1,28 +1,22 @@ -# template-rust +# reachability-toolkit -Template repository for a Rust project. +This toolkit provides, `reachability-toolkit` command line tool, +to create migration file from vulnerable batch data. -TODOs for a new project: -- [ ] Change the license if MPL2 is not appropriate for the project. Make sure to do this before adding any code. -- [ ] Set [CODEOWNERS] to the team that owns the repository. -- [ ] Create an API user in [FOSSA] and store it as a secret named `FOSSA_API_KEY`. - - Consider naming it with the pattern `ci-{REPO_NAME}`. For example, `ci-template-rust`. -- [ ] Update repository permissions as appropriate. Generally, the CODEOWNER team is set as admin. -- [ ] Update branch protection rules as appropriate. -- [ ] Update repository features and settings. Recommended defaults: - - [ ] Turn off all features (Wikis, Issues, Sponsorships, Discussions, Projects); FOSSA uses other systems for these. - - [ ] Only allow squash merging. - - [ ] Always suggest updating PR branches. - - [ ] Allow auto-merge. - - [ ] Automatically delete head branches. +## Usage -Then just edit the included Rust project, or remove it and `cargo init` your project, and get going! +```bash +./reachability-toolkit example +# ./reachability-toolkit example +# Wrote file at: example.yml -[codeowners]: https://docs.github.com/en/repositories/managing-your-repositorys-settings-and-features/customizing-your-repository/about-code-owners -[fossa]: https://app.fossa.com +./reachability-toolkit migrate example.yml +# ./reachability-toolkit migrate +# Wrote file at: 20231207203051-vulnComponent-automated.js +``` -# recommendations +## Installing + +You can install `reachability-toolkit` by downloading release +from [github release page](https://github.com/fossas/reachability-toolkit/releases). -- Prefer [cross compilation](./docs/dev/reference/cross-compile.md) over running workflows in distinct runners when possible. -- If publishing a Linux binary, consider providing two: one that [statically links libc](./docs/dev/reference/static-binary.md), and one that doesn't. -- If publishing a macOS binary, consider providing two: one for [Intel and one for M-series CPUs](./docs/dev/reference/macos-arch.md). diff --git a/src/lib.rs b/src/lib.rs index 4ee70a2..b6580b6 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,5 +1,5 @@ -#[cfg(test)] -mod tests { - #[test] - fn lib_works() {} +pub mod migration { + pub mod entry; + pub mod example; + pub mod util; } diff --git a/src/main.rs b/src/main.rs index ae16713..f98c69e 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,9 +1,44 @@ -fn main() { - println!("Hello, world!"); +use chrono::Utc; +use clap::{Parser, Subcommand}; +use reachability_toolkit::migration::{ + example::get_example, + util::{entries_from, mk_migration, write_file}, +}; +use std::path::PathBuf; + +#[derive(Parser, Debug)] +#[command(author, version, about, long_about = None, arg_required_else_help = true)] +struct Cli { + #[command(subcommand)] + command: Option, +} + +#[derive(Debug, Subcommand)] +enum Commands { + /// Create example vulnerable batch data file + Example, + + /// Create migration from vulenrable data batch file + Migrate { + // File from which to create migration file + file: PathBuf, + }, } -#[cfg(test)] -mod tests { - #[test] - fn bin_works() {} +fn main() -> anyhow::Result<()> { + let cli = Cli::parse(); + match &cli.command { + Some(Commands::Example) => write_file("example.yml", get_example()), + Some(Commands::Migrate { file }) => { + let entries = entries_from(file)?; + write_file( + &format!( + "{}-vulnComponent-automated.js", + Utc::now().format("%Y%m%d%H%M%S") + ), + &mk_migration(entries), + ) + } + _ => Ok(()), + } } diff --git a/src/migration/entry.rs b/src/migration/entry.rs new file mode 100644 index 0000000..4bbff22 --- /dev/null +++ b/src/migration/entry.rs @@ -0,0 +1,177 @@ +use anyhow::Result; +use lazy_static::lazy_static; +use regex::Regex; +use serde::de::Error; +use serde::{Deserialize, Deserializer, Serialize, Serializer}; + +#[derive(Debug, Serialize, Deserialize)] +pub struct VulnComponentBatch { + pub entries: Vec, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct VulnComponentEntry { + #[serde(deserialize_with = "non_empty_string")] + pub cve: String, + + #[serde(deserialize_with = "locator")] + pub dependency_revision_id: String, + + #[serde(serialize_with = "as_json_str")] + pub function: SymbolTarget, + + #[serde(deserialize_with = "non_empty_string")] + pub researcher: String, + pub evidence_notes: Option, + pub file_path: Option, + pub line_start: Option, +} + +// TODO: This should be using same type as reachability lib +#[derive(Debug, Serialize, Deserialize, PartialEq)] +#[serde(tag = "kind")] +pub enum SymbolTarget { + #[serde(rename = "java")] + Java { + #[serde(deserialize_with = "parse_java_symbols")] + symbol: Vec, + }, +} + +// TODO: This should be using same type as reachability lib +#[derive(Debug, Serialize, PartialEq)] +#[serde(tag = "kind")] +pub enum SymbolJava { + #[serde(rename = "package")] + Package { label: String }, + #[serde(rename = "class")] + Class { label: String }, + #[serde(rename = "class_method")] + ClassMethod { label: String }, + #[serde(rename = "constructor")] + Constructor { label: String }, +} + +fn parse_symbol_java(input: &str) -> Option { + let mut parts = input.split("::"); + let kind_label_pair = parts.next()?; + let Some((kind, label)) = kind_label_pair.split_once('(') else { + return None; + }; + + let kind = kind.trim(); + let label = label.trim_end_matches(')'); + + match kind { + "Package" => Some(SymbolJava::Package { + label: label.to_string(), + }), + "Class" => Some(SymbolJava::Class { + label: label.to_string(), + }), + "ClassMethod" => Some(SymbolJava::ClassMethod { + label: label.to_string(), + }), + "Constructor" => Some(SymbolJava::Constructor { + label: label.to_string(), + }), + _ => None, + } +} + +/// Deserialize non-empty string +/// Error when string is empty or only contains whitespace +fn non_empty_string<'de, D>(deserializer: D) -> Result +where + D: Deserializer<'de>, +{ + let s = String::deserialize(deserializer)?; + if s.is_empty() { + return Err(D::Error::custom( + "Empty string is not allowed, provide non-empty string", + )); + } + Ok(s) +} + +/// Parses FOSSA locator from non-empty string +fn locator<'de, D>(deserializer: D) -> Result +where + D: Deserializer<'de>, +{ + let s = String::deserialize(deserializer)?; + lazy_static! { + static ref RE: Regex = Regex::new(r"^[a-z]+(\+[a-zA-Z0-9_:.$-]+)$") + .expect("Locator parsing expression must compile"); + } + + if !s.is_empty() && RE.is_match(&s) { + return Ok(s); + } + + Err(Error::invalid_value( + serde::de::Unexpected::Str(s.as_str()), + &"non-empty locator: +$, e.g. pip+numpyt$1.0.0", + )) +} + +/// Parses java symbol from non-empty string +fn parse_java_symbols<'de, D>(deserializer: D) -> Result, D::Error> +where + D: Deserializer<'de>, +{ + let s = String::deserialize(deserializer)?; + if !s.is_empty() { + let symbols: Vec = s.split("::").filter_map(parse_symbol_java).collect(); + if !symbols.is_empty() { + return Ok(symbols); + } + } + + Err(Error::invalid_value( + serde::de::Unexpected::Str(s.as_str()), + &"fully qualified symbol path: Class(SomeClassName)::ClassMethod(SomeClassMethod)", + )) +} + +/// Serializes `value` as into JSON string. +fn as_json_str(value: impl Serialize, serializer: S) -> Result +where + S: Serializer, +{ + serializer.serialize_str(&serde_json::to_string(&value).map_err(serde::ser::Error::custom)?) +} + +#[cfg(test)] +mod tests { + use super::*; + use test_case::test_case; + + #[ + test_case("Class(Logger)::ClassMethod(log)", + vec![ + SymbolJava::Class {label: "Logger".to_string() }, + SymbolJava::ClassMethod { label: "log".to_string() } + ] + )] + #[ + test_case("Class(Logger)", + vec![ + SymbolJava::Class {label: "Logger".to_string() }, + ] + )] + fn parse_java_symbols_works(arg: &str, expected: Vec) { + let json_string = format!( + r#" + {{ + "kind": "java", + "symbol": "{}" + }} + "#, + arg + ); + + let symbol: SymbolTarget = serde_json::from_str(&json_string).unwrap(); + assert_eq!(symbol, SymbolTarget::Java { symbol: expected }) + } +} diff --git a/src/migration/example.rs b/src/migration/example.rs new file mode 100644 index 0000000..3f1043e --- /dev/null +++ b/src/migration/example.rs @@ -0,0 +1,14 @@ +use indoc::indoc; + +pub fn get_example() -> &'static str { + indoc! {" + entries: + - cve: 'CVE-2022-37865' # ID of the CVE + dependency_revision_id: 'mvn+org.apache.ivy:ivy' # FOSSA locator: +$ + function: # Vulnerable Function + kind: java # Kind of function. Possible choices: 'java'. + symbol: 'Class(ZipPacking)::ClassMethod(unpack)' # Qualified Function Path. Possible choices: Class(), ClassMethod(), Constructor(). `::` denotes sub scope. + researcher: 'Name of the person' # Name of the researcher + notes: 'some notes' # Any notes (Optional) + "} +} diff --git a/src/migration/util.rs b/src/migration/util.rs new file mode 100644 index 0000000..8b7d124 --- /dev/null +++ b/src/migration/util.rs @@ -0,0 +1,68 @@ +use anyhow::Context; +use anyhow::Result; +use clap::{crate_name, crate_version}; +use indoc::formatdoc; +use std::{fs::File, io::Write}; +use std::{ + io::{self, Read}, + path::PathBuf, +}; + +use crate::migration::entry::VulnComponentBatch; + +/// Write content to file. +pub fn write_file(file: &str, content: &str) -> Result<()> { + File::create(file)?.write_all(content.as_bytes())?; + println!("Wrote file at: {}", &file); + + Ok(()) +} + +/// Read and parse a YAML file containing VulnComponentBatch entries. +/// +/// This function takes a `PathBuf` representing the path to a YAML file. It opens +/// the file, reads its content, and attempts to parse it into a `VulnComponentBatch` +/// struct using serde_yaml. +pub fn entries_from(file_path: &PathBuf) -> Result { + let file = + File::open(file_path).with_context(|| format!("open file `{}`", file_path.display()))?; + + let mut content = String::new(); + let mut reader = io::BufReader::new(file); + reader + .read_to_string(&mut content) + .with_context(|| format!("read content of: `{}`", file_path.display()))?; + + serde_yaml::from_str::(&content).with_context(|| format!("parse: `{}`", file_path.display())) +} + +/// Generate a migration file for a given set of entries. +/// +/// This function takes an `Entries` struct as an argument and creates a migration file +/// with the necessary content for inserting entries into a database table. The content +/// includes information such as the app name, version, and entries in a specific format. +pub fn mk_migration(batch: VulnComponentBatch) -> String { + let entries = serde_json::to_string_pretty(&batch.entries).expect("to serialize entries"); + formatdoc! {" + // Automatically generated by: {app_name} {app_version} + + /* eslint-disable quotes */ + /* eslint-disable quote-props */ + /* eslint-disable comma-dangle */ + const entries = {entries}; + + module.exports = {{ + up: async (queryInterface, _sequelize) => {{ + queryInterface.bulkInsert('ReachabilityPrivateVulnComponents', entries, {{ + updateOnDuplicate: ['evidence_notes', 'file_path', 'line_start', 'researcher', 'updated_at'], + upsertKeys: ['cve', 'dependency_revision_id', 'function'], + }}); + }}, + down: async (_queryInterface, _) => {{}}, + }}; + ", + app_name = crate_name!(), + app_version = crate_version!(), + entries = entries, + } +} From fc5b83178368fb0cde65fdc4b3fcd29de89a15ec Mon Sep 17 00:00:00 2001 From: megh Date: Thu, 7 Dec 2023 13:50:08 -0700 Subject: [PATCH 2/9] remove srclib --- Cargo.lock | 139 +++-------------------------------------------------- Cargo.toml | 1 - 2 files changed, 6 insertions(+), 134 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0d19ab5..ad568c2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -152,7 +152,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.39", + "syn", ] [[package]] @@ -179,18 +179,6 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" -[[package]] -name = "getset" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e45727250e75cc04ff2846a66397da8ef2b3db8e40e0cef4df67950a07621eb9" -dependencies = [ - "proc-macro-error", - "proc-macro2", - "quote", - "syn 1.0.109", -] - [[package]] name = "hashbrown" version = "0.14.3" @@ -296,30 +284,6 @@ version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" -[[package]] -name = "proc-macro-error" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" -dependencies = [ - "proc-macro-error-attr", - "proc-macro2", - "quote", - "syn 1.0.109", - "version_check", -] - -[[package]] -name = "proc-macro-error-attr" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" -dependencies = [ - "proc-macro2", - "quote", - "version_check", -] - [[package]] name = "proc-macro2" version = "1.0.70" @@ -351,7 +315,6 @@ dependencies = [ "serde", "serde_json", "serde_yaml", - "srclib", "test-case", ] @@ -384,12 +347,6 @@ version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f" -[[package]] -name = "rustversion" -version = "1.0.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ffc183a10b4478d04cbbbfc96d0873219d962dd5accaff2ffbd4ceb7df837f4" - [[package]] name = "ryu" version = "1.0.15" @@ -413,7 +370,7 @@ checksum = "43576ca501357b9b071ac53cdc7da8ef0cbd9493d8df094cd821777ea6e894d3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.39", + "syn", ] [[package]] @@ -440,59 +397,12 @@ dependencies = [ "unsafe-libyaml", ] -[[package]] -name = "srclib" -version = "0.1.0" -source = "git+https://github.com/fossas/foundation-libs#4bc3762e73f371717566fb075d02e1d25b21146e" -dependencies = [ - "getset", - "lazy_static", - "regex", - "serde", - "strum", - "thiserror", - "typed-builder", -] - [[package]] name = "strsim" version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" -[[package]] -name = "strum" -version = "0.24.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "063e6045c0e62079840579a7e47a355ae92f60eb74daaf156fb1e84ba164e63f" -dependencies = [ - "strum_macros", -] - -[[package]] -name = "strum_macros" -version = "0.24.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e385be0d24f186b4ce2f9982191e7101bb737312ad61c1f2f984f34bcf85d59" -dependencies = [ - "heck", - "proc-macro2", - "quote", - "rustversion", - "syn 1.0.109", -] - -[[package]] -name = "syn" -version = "1.0.109" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - [[package]] name = "syn" version = "2.0.39" @@ -522,7 +432,7 @@ dependencies = [ "cfg-if", "proc-macro2", "quote", - "syn 2.0.39", + "syn", ] [[package]] @@ -533,41 +443,10 @@ checksum = "5c89e72a01ed4c579669add59014b9a524d609c0c88c6a585ce37485879f6ffb" dependencies = [ "proc-macro2", "quote", - "syn 2.0.39", + "syn", "test-case-core", ] -[[package]] -name = "thiserror" -version = "1.0.50" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9a7210f5c9a7156bb50aa36aed4c95afb51df0df00713949448cf9e97d382d2" -dependencies = [ - "thiserror-impl", -] - -[[package]] -name = "thiserror-impl" -version = "1.0.50" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "266b2e40bc00e5a6c09c3584011e08b06f123c00362c92b975ba9843aaaa14b8" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.39", -] - -[[package]] -name = "typed-builder" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89851716b67b937e393b3daa8423e67ddfc4bbbf1654bcf05488e95e0828db0c" -dependencies = [ - "proc-macro2", - "quote", - "syn 1.0.109", -] - [[package]] name = "unicode-ident" version = "1.0.12" @@ -586,12 +465,6 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" -[[package]] -name = "version_check" -version = "0.9.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" - [[package]] name = "wasm-bindgen" version = "0.2.89" @@ -613,7 +486,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.39", + "syn", "wasm-bindgen-shared", ] @@ -635,7 +508,7 @@ checksum = "f0eb82fcb7930ae6219a7ecfd55b217f5f0893484b7a13022ebb2b2bf20b5283" dependencies = [ "proc-macro2", "quote", - "syn 2.0.39", + "syn", "wasm-bindgen-backend", "wasm-bindgen-shared", ] diff --git a/Cargo.toml b/Cargo.toml index 86a40a3..53a84f1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,7 +13,6 @@ serde_yaml = "*" anyhow = "1.0" regex = "1" chrono = "0.4" -srclib = { git = "https://github.com/fossas/foundation-libs" } indoc = "2" lazy_static="*" From d938da2e93ada13ea27988a06e369e23dd502421 Mon Sep 17 00:00:00 2001 From: megh Date: Thu, 7 Dec 2023 13:53:43 -0700 Subject: [PATCH 3/9] update from broker to reachability-toolkit --- .github/workflows/release.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index d65f71b..9945283 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -85,7 +85,7 @@ jobs: run: | mkdir release cross build --features jemalloc --target=x86_64-unknown-linux-musl --release - mv target/x86_64-unknown-linux-musl/release/broker release/broker-$RELEASE_VERSION-x86_64-linux + mv target/x86_64-unknown-linux-musl/release/reachability-toolkit release/reachability-toolkit-$RELEASE_VERSION-x86_64-linux chmod +x release/* gh release upload ${{ github.ref_name }} $(find release -mindepth 1 | xargs) --clobber @@ -97,8 +97,8 @@ jobs: rustup target add x86_64-apple-darwin cargo build --target=aarch64-apple-darwin --release cargo build --target=x86_64-apple-darwin --release - mv target/aarch64-apple-darwin/release/broker release/broker-$RELEASE_VERSION-aarch64-macos - mv target/x86_64-apple-darwin/release/broker release/broker-$RELEASE_VERSION-x86_64-macos + mv target/aarch64-apple-darwin/release/reachability-toolkit release/reachability-toolkit-$RELEASE_VERSION-aarch64-macos + mv target/x86_64-apple-darwin/release/reachability-toolkit release/reachability-toolkit-$RELEASE_VERSION-x86_64-macos chmod +x release/* gh release upload ${{ github.ref_name }} $(find release -mindepth 1 | xargs) --clobber @@ -107,8 +107,8 @@ jobs: run: | mkdir release cargo build --release - mv target/release/broker.exe release/broker-$env:RELEASE_VERSION-x86_64-windows.exe - gh release upload ${{ github.ref_name }} release/broker-$env:RELEASE_VERSION-x86_64-windows.exe --clobber + mv target/release/reachability-toolkit.exe release/reachability-toolkit-$env:RELEASE_VERSION-x86_64-windows.exe + gh release upload ${{ github.ref_name }} release/reachability-toolkit-$env:RELEASE_VERSION-x86_64-windows.exe --clobber publish-release: needs: [metadata, build] From b6620da958ec7e20a447a70b0cfaeed716f7f91e Mon Sep 17 00:00:00 2001 From: megh Date: Thu, 7 Dec 2023 15:51:12 -0700 Subject: [PATCH 4/9] use locator-rs --- Cargo.lock | 140 ++++++++++++++++++++++++++++++++++++++--- Cargo.toml | 2 +- src/migration/entry.rs | 28 +-------- 3 files changed, 136 insertions(+), 34 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ad568c2..f17a0f5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -152,7 +152,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn", + "syn 2.0.39", ] [[package]] @@ -179,6 +179,18 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" +[[package]] +name = "getset" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e45727250e75cc04ff2846a66397da8ef2b3db8e40e0cef4df67950a07621eb9" +dependencies = [ + "proc-macro-error", + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "hashbrown" version = "0.14.3" @@ -257,6 +269,20 @@ version = "0.2.150" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "89d92a4743f9a61002fae18374ed11e7973f530cb3a3255fb354818118b2203c" +[[package]] +name = "locator" +version = "1.0.0" +source = "git+https://github.com/fossas/locator-rs.git#81f7cc2efc1ef93053cce6146d11fb489958cec2" +dependencies = [ + "getset", + "lazy_static", + "regex", + "serde", + "strum", + "thiserror", + "typed-builder", +] + [[package]] name = "log" version = "0.4.20" @@ -284,6 +310,30 @@ version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" +[[package]] +name = "proc-macro-error" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" +dependencies = [ + "proc-macro-error-attr", + "proc-macro2", + "quote", + "syn 1.0.109", + "version_check", +] + +[[package]] +name = "proc-macro-error-attr" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" +dependencies = [ + "proc-macro2", + "quote", + "version_check", +] + [[package]] name = "proc-macro2" version = "1.0.70" @@ -310,7 +360,7 @@ dependencies = [ "chrono", "clap", "indoc", - "lazy_static", + "locator", "regex", "serde", "serde_json", @@ -347,6 +397,12 @@ version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f" +[[package]] +name = "rustversion" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ffc183a10b4478d04cbbbfc96d0873219d962dd5accaff2ffbd4ceb7df837f4" + [[package]] name = "ryu" version = "1.0.15" @@ -370,7 +426,7 @@ checksum = "43576ca501357b9b071ac53cdc7da8ef0cbd9493d8df094cd821777ea6e894d3" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.39", ] [[package]] @@ -403,6 +459,39 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" +[[package]] +name = "strum" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "063e6045c0e62079840579a7e47a355ae92f60eb74daaf156fb1e84ba164e63f" +dependencies = [ + "strum_macros", +] + +[[package]] +name = "strum_macros" +version = "0.24.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e385be0d24f186b4ce2f9982191e7101bb737312ad61c1f2f984f34bcf85d59" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "rustversion", + "syn 1.0.109", +] + +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "syn" version = "2.0.39" @@ -432,7 +521,7 @@ dependencies = [ "cfg-if", "proc-macro2", "quote", - "syn", + "syn 2.0.39", ] [[package]] @@ -443,10 +532,41 @@ checksum = "5c89e72a01ed4c579669add59014b9a524d609c0c88c6a585ce37485879f6ffb" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.39", "test-case-core", ] +[[package]] +name = "thiserror" +version = "1.0.50" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9a7210f5c9a7156bb50aa36aed4c95afb51df0df00713949448cf9e97d382d2" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.50" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "266b2e40bc00e5a6c09c3584011e08b06f123c00362c92b975ba9843aaaa14b8" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.39", +] + +[[package]] +name = "typed-builder" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89851716b67b937e393b3daa8423e67ddfc4bbbf1654bcf05488e95e0828db0c" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "unicode-ident" version = "1.0.12" @@ -465,6 +585,12 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" +[[package]] +name = "version_check" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" + [[package]] name = "wasm-bindgen" version = "0.2.89" @@ -486,7 +612,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn", + "syn 2.0.39", "wasm-bindgen-shared", ] @@ -508,7 +634,7 @@ checksum = "f0eb82fcb7930ae6219a7ecfd55b217f5f0893484b7a13022ebb2b2bf20b5283" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.39", "wasm-bindgen-backend", "wasm-bindgen-shared", ] diff --git a/Cargo.toml b/Cargo.toml index 53a84f1..7b87a64 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,7 +14,7 @@ anyhow = "1.0" regex = "1" chrono = "0.4" indoc = "2" -lazy_static="*" +locator = { git = "https://github.com/fossas/locator-rs.git"} [dev-dependencies] test-case = "*" diff --git a/src/migration/entry.rs b/src/migration/entry.rs index 4bbff22..653eab9 100644 --- a/src/migration/entry.rs +++ b/src/migration/entry.rs @@ -1,6 +1,5 @@ use anyhow::Result; -use lazy_static::lazy_static; -use regex::Regex; +use locator::Locator; use serde::de::Error; use serde::{Deserialize, Deserializer, Serialize, Serializer}; @@ -13,9 +12,7 @@ pub struct VulnComponentBatch { pub struct VulnComponentEntry { #[serde(deserialize_with = "non_empty_string")] pub cve: String, - - #[serde(deserialize_with = "locator")] - pub dependency_revision_id: String, + pub dependency_revision_id: Locator, #[serde(serialize_with = "as_json_str")] pub function: SymbolTarget, @@ -94,27 +91,6 @@ where Ok(s) } -/// Parses FOSSA locator from non-empty string -fn locator<'de, D>(deserializer: D) -> Result -where - D: Deserializer<'de>, -{ - let s = String::deserialize(deserializer)?; - lazy_static! { - static ref RE: Regex = Regex::new(r"^[a-z]+(\+[a-zA-Z0-9_:.$-]+)$") - .expect("Locator parsing expression must compile"); - } - - if !s.is_empty() && RE.is_match(&s) { - return Ok(s); - } - - Err(Error::invalid_value( - serde::de::Unexpected::Str(s.as_str()), - &"non-empty locator: +$, e.g. pip+numpyt$1.0.0", - )) -} - /// Parses java symbol from non-empty string fn parse_java_symbols<'de, D>(deserializer: D) -> Result, D::Error> where From 8b649c18e4b2fa938ba96999e81942de6719365e Mon Sep 17 00:00:00 2001 From: Megh Date: Mon, 11 Dec 2023 15:49:54 -0700 Subject: [PATCH 5/9] address pr feedback --- Cargo.lock | 31 +++++- Cargo.toml | 9 +- src/lib.rs | 6 +- src/main.rs | 22 +--- src/migration.rs | 214 +++++++++++++++++++++++++++++++++++++++ src/migration/entry.rs | 153 ---------------------------- src/migration/example.rs | 14 --- src/migration/util.rs | 68 ------------- 8 files changed, 255 insertions(+), 262 deletions(-) create mode 100644 src/migration.rs delete mode 100644 src/migration/entry.rs delete mode 100644 src/migration/example.rs delete mode 100644 src/migration/util.rs diff --git a/Cargo.lock b/Cargo.lock index f17a0f5..5b23a9a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -173,6 +173,17 @@ version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" +[[package]] +name = "delegate" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "082a24a9967533dc5d743c602157637116fc1b52806d694a5a45e6f32567fcdd" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "equivalent" version = "1.0.1" @@ -272,7 +283,7 @@ checksum = "89d92a4743f9a61002fae18374ed11e7973f530cb3a3255fb354818118b2203c" [[package]] name = "locator" version = "1.0.0" -source = "git+https://github.com/fossas/locator-rs.git#81f7cc2efc1ef93053cce6146d11fb489958cec2" +source = "git+https://github.com/fossas/locator-rs.git?branch=main#81f7cc2efc1ef93053cce6146d11fb489958cec2" dependencies = [ "getset", "lazy_static", @@ -295,6 +306,16 @@ version = "2.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f665ee40bc4a3c5590afb1e9677db74a508659dfd71e126420da8274909a0167" +[[package]] +name = "non-empty-string" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55cf0f4060e345ae505219853da9ca1150564158a648a6aa6a528f0d5794bb33" +dependencies = [ + "delegate", + "serde", +] + [[package]] name = "num-traits" version = "0.2.17" @@ -361,10 +382,12 @@ dependencies = [ "clap", "indoc", "locator", + "non-empty-string", "regex", "serde", "serde_json", "serde_yaml", + "tap", "test-case", ] @@ -503,6 +526,12 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "tap" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" + [[package]] name = "test-case" version = "3.3.1" diff --git a/Cargo.toml b/Cargo.toml index 7b87a64..7c087a6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,18 +3,21 @@ name = "reachability-toolkit" version = "0.1.0" edition = "2021" authors = ["github.com/fossas"] +publish = false # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] clap = { version = "4.0.26", features = ["derive", "cargo"] } serde = { version = "1.0.147", features = ["derive"] } serde_json = "1.0.88" -serde_yaml = "*" +serde_yaml = "0.9.27" anyhow = "1.0" regex = "1" chrono = "0.4" indoc = "2" -locator = { git = "https://github.com/fossas/locator-rs.git"} +locator = { git = "https://github.com/fossas/locator-rs.git", branch="main"} +non-empty-string = { version="0.2.4", features= ["serde"] } +tap = "1.0.1" [dev-dependencies] -test-case = "*" +test-case = "3.3.1" diff --git a/src/lib.rs b/src/lib.rs index b6580b6..a571eaa 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,5 +1 @@ -pub mod migration { - pub mod entry; - pub mod example; - pub mod util; -} +pub mod migration; diff --git a/src/main.rs b/src/main.rs index f98c69e..cd56566 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,16 +1,12 @@ -use chrono::Utc; use clap::{Parser, Subcommand}; -use reachability_toolkit::migration::{ - example::get_example, - util::{entries_from, mk_migration, write_file}, -}; +use reachability_toolkit::migration::{Migrate, VulnComponentBatch}; use std::path::PathBuf; #[derive(Parser, Debug)] #[command(author, version, about, long_about = None, arg_required_else_help = true)] struct Cli { #[command(subcommand)] - command: Option, + command: Commands, } #[derive(Debug, Subcommand)] @@ -28,17 +24,7 @@ enum Commands { fn main() -> anyhow::Result<()> { let cli = Cli::parse(); match &cli.command { - Some(Commands::Example) => write_file("example.yml", get_example()), - Some(Commands::Migrate { file }) => { - let entries = entries_from(file)?; - write_file( - &format!( - "{}-vulnComponent-automated.js", - Utc::now().format("%Y%m%d%H%M%S") - ), - &mk_migration(entries), - ) - } - _ => Ok(()), + Commands::Example => VulnComponentBatch::make_example(&PathBuf::from("example.yml")), + Commands::Migrate { file } => VulnComponentBatch::make_migration(file), } } diff --git a/src/migration.rs b/src/migration.rs new file mode 100644 index 0000000..4a70265 --- /dev/null +++ b/src/migration.rs @@ -0,0 +1,214 @@ +use std::fs::File; +use std::io::{self, Read, Write}; +use std::path::Path; + +use anyhow::{Context, Result}; +use chrono::Utc; +use clap::{crate_name, crate_version}; +use indoc::{formatdoc, indoc}; +use locator::Locator; +use non_empty_string::NonEmptyString; +use serde::de::Error; +use serde::{Deserialize, Serialize, Serializer}; +use tap::TapFallible; + +#[derive(Debug, Serialize, Deserialize)] +pub struct VulnComponentBatch { + pub entries: Vec, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct VulnComponentEntry { + pub cve: NonEmptyString, + pub dependency_revision_id: Locator, + + #[serde(serialize_with = "as_json_str")] + pub function: SymbolTarget, + pub researcher: NonEmptyString, + pub evidence_notes: Option, + pub file_path: Option, + pub line_start: Option, +} + +#[derive(Debug, Serialize, PartialEq)] +pub struct JavaSymbols(Vec); + +// TODO: This should be using same type as reachability lib +#[derive(Debug, Serialize, Deserialize, PartialEq)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub enum SymbolTarget { + Java { symbol: JavaSymbols }, +} + +// TODO: This should be using same type as reachability lib +#[derive(Debug, Serialize, PartialEq)] +#[serde(rename_all = "snake_case", tag = "kind")] +pub enum SymbolJava { + Package { label: String }, + Class { label: String }, + ClassMethod { label: String }, + Constructor { label: String }, +} + +impl<'de> Deserialize<'de> for JavaSymbols { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + let s = String::deserialize(deserializer)?; + if !s.is_empty() { + let symbols: Vec = s.split("::").filter_map(parse_symbol_java).collect(); + if !symbols.is_empty() { + return Ok(JavaSymbols(symbols)); + } + } + + Err(Error::invalid_value( + serde::de::Unexpected::Str(s.as_str()), + &"fully qualified symbol path: Class(SomeClassName)::ClassMethod(SomeClassMethod)", + )) + } +} + +pub trait Migrate { + fn make_migration(file: &Path) -> Result<()>; + fn make_example(file: &Path) -> Result<()>; +} + +impl Migrate for VulnComponentBatch { + fn make_example(out_file: &Path) -> Result<()> { + let content = indoc! {" + entries: + - cve: 'CVE-2022-37865' # ID of the CVE + dependency_revision_id: 'mvn+org.apache.ivy:ivy' # FOSSA locator: +$ + function: # Vulnerable Function + kind: java # Kind of function. Possible choices: 'java'. + symbol: 'Class(ZipPacking)::ClassMethod(unpack)' # Qualified Function Path. Possible choices: Class(), ClassMethod(), Constructor(). `::` denotes sub scope. + researcher: 'Name of the person' # Name of the researcher + notes: 'some notes' # Any notes (Optional) + "}; + + File::create(out_file) + .context("creating file")? + .write_all(content.as_bytes()) + .context("writing file") + .tap_ok(|_| println!("Wrote file at: {:?}", out_file)) + } + + fn make_migration(batch_file: &Path) -> Result<()> { + // 1. Read file + let file = + File::open(batch_file).with_context(|| format!("open file `{:?}`", batch_file))?; + let mut content = String::new(); + let mut reader = io::BufReader::new(file); + + reader + .read_to_string(&mut content) + .with_context(|| format!("read content of: `{:?}`", batch_file))?; + + let batch: VulnComponentBatch = + serde_yaml::from_str(&content).with_context(|| format!("parse: `{:?}`", batch_file))?; + + // 2. Create migration file's content + let entries: String = + serde_json::to_string_pretty(&batch.entries).expect("to serialize entries"); + let content = formatdoc! {" + // Automatically generated by: {app_name} {app_version} + /* eslint-disable quotes */ + /* eslint-disable quote-props */ + /* eslint-disable comma-dangle */ + const entries = {entries}; + + module.exports = {{ + up: async (queryInterface, _sequelize) => {{ + queryInterface.bulkInsert('ReachabilityPrivateVulnComponents', entries, {{ + updateOnDuplicate: ['evidence_notes', 'file_path', 'line_start', 'researcher', 'updated_at'], + upsertKeys: ['cve', 'dependency_revision_id', 'function'], + }}); + }}, + down: async (_queryInterface, _) => {{}}, + }}; + ", + app_name = crate_name!(), + app_version = crate_version!(), + }; + + // 3. Write migration file! + let out_file = &format!( + "{}-vulnComponent-automated.js", + Utc::now().format("%Y%m%d%H%M%S") + ); + File::create(out_file) + .context("creating file")? + .write_all(content.as_bytes()) + .context("writing file") + .tap_ok(|_| println!("Wrote file at: {:?}", out_file)) + } +} + +fn parse_symbol_java(input: &str) -> Option { + let mut parts = input.split("::"); + let kind_label_pair = parts.next()?; + let Some((kind, label)) = kind_label_pair.split_once('(') else { + return None; + }; + + let kind = kind.trim(); + let label = label.trim_end_matches(')'); + + match kind { + "Package" => Some(SymbolJava::Package { + label: label.to_string(), + }), + "Class" => Some(SymbolJava::Class { + label: label.to_string(), + }), + "ClassMethod" => Some(SymbolJava::ClassMethod { + label: label.to_string(), + }), + "Constructor" => Some(SymbolJava::Constructor { + label: label.to_string(), + }), + _ => None, + } +} + +/// Serializes `value` as into JSON string. +fn as_json_str(value: impl Serialize, serializer: S) -> Result +where + S: Serializer, +{ + serializer.serialize_str(&serde_json::to_string(&value).map_err(serde::ser::Error::custom)?) +} + +#[cfg(test)] +mod tests { + use crate::migration; + + use super::*; + use test_case::test_case; + + #[ + test_case("Class(Logger)::ClassMethod(log)", + vec![ + SymbolJava::Class {label: "Logger".to_string() }, + SymbolJava::ClassMethod { label: "log".to_string() } + ] + )] + #[ + test_case("Class(Logger)", + vec![ + SymbolJava::Class {label: "Logger".to_string() }, + ] + )] + fn parse_java_symbols_works(arg: &str, expected: Vec) { + let json_string = format!(r#"{{ "kind": "java", "symbol": "{arg}" }}"#); + let symbol: SymbolTarget = serde_json::from_str(&json_string).unwrap(); + assert_eq!( + symbol, + SymbolTarget::Java { + symbol: migration::JavaSymbols(expected) + } + ) + } +} diff --git a/src/migration/entry.rs b/src/migration/entry.rs deleted file mode 100644 index 653eab9..0000000 --- a/src/migration/entry.rs +++ /dev/null @@ -1,153 +0,0 @@ -use anyhow::Result; -use locator::Locator; -use serde::de::Error; -use serde::{Deserialize, Deserializer, Serialize, Serializer}; - -#[derive(Debug, Serialize, Deserialize)] -pub struct VulnComponentBatch { - pub entries: Vec, -} - -#[derive(Debug, Serialize, Deserialize)] -pub struct VulnComponentEntry { - #[serde(deserialize_with = "non_empty_string")] - pub cve: String, - pub dependency_revision_id: Locator, - - #[serde(serialize_with = "as_json_str")] - pub function: SymbolTarget, - - #[serde(deserialize_with = "non_empty_string")] - pub researcher: String, - pub evidence_notes: Option, - pub file_path: Option, - pub line_start: Option, -} - -// TODO: This should be using same type as reachability lib -#[derive(Debug, Serialize, Deserialize, PartialEq)] -#[serde(tag = "kind")] -pub enum SymbolTarget { - #[serde(rename = "java")] - Java { - #[serde(deserialize_with = "parse_java_symbols")] - symbol: Vec, - }, -} - -// TODO: This should be using same type as reachability lib -#[derive(Debug, Serialize, PartialEq)] -#[serde(tag = "kind")] -pub enum SymbolJava { - #[serde(rename = "package")] - Package { label: String }, - #[serde(rename = "class")] - Class { label: String }, - #[serde(rename = "class_method")] - ClassMethod { label: String }, - #[serde(rename = "constructor")] - Constructor { label: String }, -} - -fn parse_symbol_java(input: &str) -> Option { - let mut parts = input.split("::"); - let kind_label_pair = parts.next()?; - let Some((kind, label)) = kind_label_pair.split_once('(') else { - return None; - }; - - let kind = kind.trim(); - let label = label.trim_end_matches(')'); - - match kind { - "Package" => Some(SymbolJava::Package { - label: label.to_string(), - }), - "Class" => Some(SymbolJava::Class { - label: label.to_string(), - }), - "ClassMethod" => Some(SymbolJava::ClassMethod { - label: label.to_string(), - }), - "Constructor" => Some(SymbolJava::Constructor { - label: label.to_string(), - }), - _ => None, - } -} - -/// Deserialize non-empty string -/// Error when string is empty or only contains whitespace -fn non_empty_string<'de, D>(deserializer: D) -> Result -where - D: Deserializer<'de>, -{ - let s = String::deserialize(deserializer)?; - if s.is_empty() { - return Err(D::Error::custom( - "Empty string is not allowed, provide non-empty string", - )); - } - Ok(s) -} - -/// Parses java symbol from non-empty string -fn parse_java_symbols<'de, D>(deserializer: D) -> Result, D::Error> -where - D: Deserializer<'de>, -{ - let s = String::deserialize(deserializer)?; - if !s.is_empty() { - let symbols: Vec = s.split("::").filter_map(parse_symbol_java).collect(); - if !symbols.is_empty() { - return Ok(symbols); - } - } - - Err(Error::invalid_value( - serde::de::Unexpected::Str(s.as_str()), - &"fully qualified symbol path: Class(SomeClassName)::ClassMethod(SomeClassMethod)", - )) -} - -/// Serializes `value` as into JSON string. -fn as_json_str(value: impl Serialize, serializer: S) -> Result -where - S: Serializer, -{ - serializer.serialize_str(&serde_json::to_string(&value).map_err(serde::ser::Error::custom)?) -} - -#[cfg(test)] -mod tests { - use super::*; - use test_case::test_case; - - #[ - test_case("Class(Logger)::ClassMethod(log)", - vec![ - SymbolJava::Class {label: "Logger".to_string() }, - SymbolJava::ClassMethod { label: "log".to_string() } - ] - )] - #[ - test_case("Class(Logger)", - vec![ - SymbolJava::Class {label: "Logger".to_string() }, - ] - )] - fn parse_java_symbols_works(arg: &str, expected: Vec) { - let json_string = format!( - r#" - {{ - "kind": "java", - "symbol": "{}" - }} - "#, - arg - ); - - let symbol: SymbolTarget = serde_json::from_str(&json_string).unwrap(); - assert_eq!(symbol, SymbolTarget::Java { symbol: expected }) - } -} diff --git a/src/migration/example.rs b/src/migration/example.rs deleted file mode 100644 index 3f1043e..0000000 --- a/src/migration/example.rs +++ /dev/null @@ -1,14 +0,0 @@ -use indoc::indoc; - -pub fn get_example() -> &'static str { - indoc! {" - entries: - - cve: 'CVE-2022-37865' # ID of the CVE - dependency_revision_id: 'mvn+org.apache.ivy:ivy' # FOSSA locator: +$ - function: # Vulnerable Function - kind: java # Kind of function. Possible choices: 'java'. - symbol: 'Class(ZipPacking)::ClassMethod(unpack)' # Qualified Function Path. Possible choices: Class(), ClassMethod(), Constructor(). `::` denotes sub scope. - researcher: 'Name of the person' # Name of the researcher - notes: 'some notes' # Any notes (Optional) - "} -} diff --git a/src/migration/util.rs b/src/migration/util.rs deleted file mode 100644 index 8b7d124..0000000 --- a/src/migration/util.rs +++ /dev/null @@ -1,68 +0,0 @@ -use anyhow::Context; -use anyhow::Result; -use clap::{crate_name, crate_version}; -use indoc::formatdoc; -use std::{fs::File, io::Write}; -use std::{ - io::{self, Read}, - path::PathBuf, -}; - -use crate::migration::entry::VulnComponentBatch; - -/// Write content to file. -pub fn write_file(file: &str, content: &str) -> Result<()> { - File::create(file)?.write_all(content.as_bytes())?; - println!("Wrote file at: {}", &file); - - Ok(()) -} - -/// Read and parse a YAML file containing VulnComponentBatch entries. -/// -/// This function takes a `PathBuf` representing the path to a YAML file. It opens -/// the file, reads its content, and attempts to parse it into a `VulnComponentBatch` -/// struct using serde_yaml. -pub fn entries_from(file_path: &PathBuf) -> Result { - let file = - File::open(file_path).with_context(|| format!("open file `{}`", file_path.display()))?; - - let mut content = String::new(); - let mut reader = io::BufReader::new(file); - reader - .read_to_string(&mut content) - .with_context(|| format!("read content of: `{}`", file_path.display()))?; - - serde_yaml::from_str::(&content).with_context(|| format!("parse: `{}`", file_path.display())) -} - -/// Generate a migration file for a given set of entries. -/// -/// This function takes an `Entries` struct as an argument and creates a migration file -/// with the necessary content for inserting entries into a database table. The content -/// includes information such as the app name, version, and entries in a specific format. -pub fn mk_migration(batch: VulnComponentBatch) -> String { - let entries = serde_json::to_string_pretty(&batch.entries).expect("to serialize entries"); - formatdoc! {" - // Automatically generated by: {app_name} {app_version} - - /* eslint-disable quotes */ - /* eslint-disable quote-props */ - /* eslint-disable comma-dangle */ - const entries = {entries}; - - module.exports = {{ - up: async (queryInterface, _sequelize) => {{ - queryInterface.bulkInsert('ReachabilityPrivateVulnComponents', entries, {{ - updateOnDuplicate: ['evidence_notes', 'file_path', 'line_start', 'researcher', 'updated_at'], - upsertKeys: ['cve', 'dependency_revision_id', 'function'], - }}); - }}, - down: async (_queryInterface, _) => {{}}, - }}; - ", - app_name = crate_name!(), - app_version = crate_version!(), - entries = entries, - } -} From c8b5de1a1f88b40934ce294182819c16beae3b89 Mon Sep 17 00:00:00 2001 From: Megh Date: Mon, 11 Dec 2023 15:52:17 -0700 Subject: [PATCH 6/9] unwrap to expect --- src/migration.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/migration.rs b/src/migration.rs index 4a70265..5a9ec9c 100644 --- a/src/migration.rs +++ b/src/migration.rs @@ -203,7 +203,7 @@ mod tests { )] fn parse_java_symbols_works(arg: &str, expected: Vec) { let json_string = format!(r#"{{ "kind": "java", "symbol": "{arg}" }}"#); - let symbol: SymbolTarget = serde_json::from_str(&json_string).unwrap(); + let symbol: SymbolTarget = serde_json::from_str(&json_string).expect("to serialize into json string"); assert_eq!( symbol, SymbolTarget::Java { From 4164ac2d66c9e194694c94f0a46d4449d5317c90 Mon Sep 17 00:00:00 2001 From: Megh Date: Mon, 11 Dec 2023 15:58:13 -0700 Subject: [PATCH 7/9] fmt fix --- src/migration.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/migration.rs b/src/migration.rs index 5a9ec9c..46405a5 100644 --- a/src/migration.rs +++ b/src/migration.rs @@ -203,7 +203,8 @@ mod tests { )] fn parse_java_symbols_works(arg: &str, expected: Vec) { let json_string = format!(r#"{{ "kind": "java", "symbol": "{arg}" }}"#); - let symbol: SymbolTarget = serde_json::from_str(&json_string).expect("to serialize into json string"); + let symbol: SymbolTarget = + serde_json::from_str(&json_string).expect("to serialize into json string"); assert_eq!( symbol, SymbolTarget::Java { From 6adc7d80f85b7a5ca41d63d4b690b49d01470a25 Mon Sep 17 00:00:00 2001 From: Megh Date: Mon, 11 Dec 2023 20:06:17 -0700 Subject: [PATCH 8/9] use two modules and TryFrom<> --- src/ingestion.rs | 221 +++++++++++++++++++++++++++++++++++++++++++++++ src/lib.rs | 1 + src/main.rs | 2 +- src/migration.rs | 207 +------------------------------------------- 4 files changed, 226 insertions(+), 205 deletions(-) create mode 100644 src/ingestion.rs diff --git a/src/ingestion.rs b/src/ingestion.rs new file mode 100644 index 0000000..31606cd --- /dev/null +++ b/src/ingestion.rs @@ -0,0 +1,221 @@ +use std::fs::File; +use std::io::{self, Read, Write}; +use std::path::Path; + +use anyhow::{Context, Result}; +use chrono::Utc; +use clap::{crate_name, crate_version}; +use indoc::{formatdoc, indoc}; +use locator::Locator; +use non_empty_string::NonEmptyString; +use serde::de::Error; +use serde::{Deserialize, Serialize}; +use tap::TapFallible; + +use crate::migration::MigrationVulnComponentEntry; + +#[derive(Debug, Serialize, Deserialize)] +pub struct VulnComponentBatch { + pub entries: Vec, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct VulnComponentEntry { + pub cve: NonEmptyString, + pub dependency_revision_id: Locator, + pub function: SymbolTarget, + pub researcher: NonEmptyString, + pub evidence_notes: Option, + pub file_path: Option, + pub line_start: Option, +} + +impl TryFrom for MigrationVulnComponentEntry { + type Error = serde_json::Error; + fn try_from(v: VulnComponentEntry) -> Result { + Ok(Self { + cve: v.cve, + dependency_revision_id: v.dependency_revision_id, + function: serde_json::to_string(&v.function)?, + researcher: v.researcher, + evidence_notes: v.evidence_notes, + file_path: v.file_path, + line_start: v.line_start, + }) + } +} + +#[derive(Debug, Serialize, PartialEq)] +pub struct JavaSymbols(Vec); + +// TODO: This should be using same type as reachability lib +#[derive(Debug, Serialize, Deserialize, PartialEq)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub enum SymbolTarget { + Java { symbol: JavaSymbols }, +} + +// TODO: This should be using same type as reachability lib +#[derive(Debug, Serialize, PartialEq)] +#[serde(rename_all = "snake_case", tag = "kind")] +pub enum SymbolJava { + Package { label: String }, + Class { label: String }, + ClassMethod { label: String }, + Constructor { label: String }, +} + +impl<'de> Deserialize<'de> for JavaSymbols { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + let s = String::deserialize(deserializer)?; + if !s.is_empty() { + let symbols: Vec = s.split("::").filter_map(parse_symbol_java).collect(); + if !symbols.is_empty() { + return Ok(JavaSymbols(symbols)); + } + } + + Err(Error::invalid_value( + serde::de::Unexpected::Str(s.as_str()), + &"fully qualified symbol path: Class(SomeClassName)::ClassMethod(SomeClassMethod)", + )) + } +} + +impl VulnComponentBatch { + pub fn make_example(out_file: &Path) -> Result<()> { + let content = indoc! {" + entries: + - cve: 'CVE-2022-37865' # ID of the CVE + dependency_revision_id: 'mvn+org.apache.ivy:ivy' # FOSSA locator: +$ + function: # Vulnerable Function + kind: java # Kind of function. Possible choices: 'java'. + symbol: 'Class(ZipPacking)::ClassMethod(unpack)' # Qualified Function Path. Possible choices: Class(), ClassMethod(), Constructor(). `::` denotes sub scope. + researcher: 'Name of the person' # Name of the researcher + notes: 'some notes' # Any notes (Optional) + "}; + + File::create(out_file) + .context("creating file")? + .write_all(content.as_bytes()) + .context("writing file") + .tap_ok(|_| println!("Wrote file at: {:?}", out_file)) + } + + pub fn make_migration(batch_file: &Path) -> Result<()> { + // 1. Read file + let file = + File::open(batch_file).with_context(|| format!("open file `{:?}`", batch_file))?; + let mut content = String::new(); + let mut reader = io::BufReader::new(file); + + reader + .read_to_string(&mut content) + .with_context(|| format!("read content of: `{:?}`", batch_file))?; + + let batch: VulnComponentBatch = + serde_yaml::from_str(&content).with_context(|| format!("parse: `{:?}`", batch_file))?; + let entries = batch + .entries + .into_iter() + .map(MigrationVulnComponentEntry::try_from) + .collect::, _>>() + .context("convert to output format")?; + + // 2. Create migration file's content + let entries: String = + serde_json::to_string_pretty(&entries).context("serialize entries")?; + let content = formatdoc! {" + // Automatically generated by: {app_name} {app_version} + /* eslint-disable quotes */ + /* eslint-disable quote-props */ + /* eslint-disable comma-dangle */ + const entries = {entries}; + + module.exports = {{ + up: async (queryInterface, _sequelize) => {{ + queryInterface.bulkInsert('ReachabilityPrivateVulnComponents', entries, {{ + updateOnDuplicate: ['evidence_notes', 'file_path', 'line_start', 'researcher', 'updated_at'], + upsertKeys: ['cve', 'dependency_revision_id', 'function'], + }}); + }}, + down: async (_queryInterface, _) => {{}}, + }}; + ", + app_name = crate_name!(), + app_version = crate_version!(), + }; + + // 3. Write migration file! + let out_file = &format!( + "{}-vulnComponent-automated.js", + Utc::now().format("%Y%m%d%H%M%S") + ); + File::create(out_file) + .context("creating file")? + .write_all(content.as_bytes()) + .context("writing file") + .tap_ok(|_| println!("Wrote file at: {:?}", out_file)) + } +} + +fn parse_symbol_java(input: &str) -> Option { + let mut parts = input.split("::"); + let kind_label_pair = parts.next()?; + let Some((kind, label)) = kind_label_pair.split_once('(') else { + return None; + }; + + let kind = kind.trim(); + let label = label.trim_end_matches(')'); + + match kind { + "Package" => Some(SymbolJava::Package { + label: label.to_string(), + }), + "Class" => Some(SymbolJava::Class { + label: label.to_string(), + }), + "ClassMethod" => Some(SymbolJava::ClassMethod { + label: label.to_string(), + }), + "Constructor" => Some(SymbolJava::Constructor { + label: label.to_string(), + }), + _ => None, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use test_case::test_case; + + #[ + test_case("Class(Logger)::ClassMethod(log)", + vec![ + SymbolJava::Class {label: "Logger".to_string() }, + SymbolJava::ClassMethod { label: "log".to_string() } + ] + )] + #[ + test_case("Class(Logger)", + vec![ + SymbolJava::Class {label: "Logger".to_string() }, + ] + )] + fn parse_java_symbols_works(arg: &str, expected: Vec) { + let json_string = format!(r#"{{ "kind": "java", "symbol": "{arg}" }}"#); + let symbol: SymbolTarget = + serde_json::from_str(&json_string).expect("to serialize into json string"); + assert_eq!( + symbol, + SymbolTarget::Java { + symbol: JavaSymbols(expected) + } + ) + } +} diff --git a/src/lib.rs b/src/lib.rs index a571eaa..95da340 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1 +1,2 @@ +pub mod ingestion; pub mod migration; diff --git a/src/main.rs b/src/main.rs index cd56566..a87f76d 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,5 +1,5 @@ use clap::{Parser, Subcommand}; -use reachability_toolkit::migration::{Migrate, VulnComponentBatch}; +use reachability_toolkit::ingestion::VulnComponentBatch; use std::path::PathBuf; #[derive(Parser, Debug)] diff --git a/src/migration.rs b/src/migration.rs index 46405a5..68655e6 100644 --- a/src/migration.rs +++ b/src/migration.rs @@ -1,215 +1,14 @@ -use std::fs::File; -use std::io::{self, Read, Write}; -use std::path::Path; - -use anyhow::{Context, Result}; -use chrono::Utc; -use clap::{crate_name, crate_version}; -use indoc::{formatdoc, indoc}; use locator::Locator; use non_empty_string::NonEmptyString; -use serde::de::Error; -use serde::{Deserialize, Serialize, Serializer}; -use tap::TapFallible; +use serde::{Deserialize, Serialize}; #[derive(Debug, Serialize, Deserialize)] -pub struct VulnComponentBatch { - pub entries: Vec, -} - -#[derive(Debug, Serialize, Deserialize)] -pub struct VulnComponentEntry { +pub struct MigrationVulnComponentEntry { pub cve: NonEmptyString, pub dependency_revision_id: Locator, - - #[serde(serialize_with = "as_json_str")] - pub function: SymbolTarget, + pub function: String, pub researcher: NonEmptyString, pub evidence_notes: Option, pub file_path: Option, pub line_start: Option, } - -#[derive(Debug, Serialize, PartialEq)] -pub struct JavaSymbols(Vec); - -// TODO: This should be using same type as reachability lib -#[derive(Debug, Serialize, Deserialize, PartialEq)] -#[serde(tag = "kind", rename_all = "snake_case")] -pub enum SymbolTarget { - Java { symbol: JavaSymbols }, -} - -// TODO: This should be using same type as reachability lib -#[derive(Debug, Serialize, PartialEq)] -#[serde(rename_all = "snake_case", tag = "kind")] -pub enum SymbolJava { - Package { label: String }, - Class { label: String }, - ClassMethod { label: String }, - Constructor { label: String }, -} - -impl<'de> Deserialize<'de> for JavaSymbols { - fn deserialize(deserializer: D) -> Result - where - D: serde::Deserializer<'de>, - { - let s = String::deserialize(deserializer)?; - if !s.is_empty() { - let symbols: Vec = s.split("::").filter_map(parse_symbol_java).collect(); - if !symbols.is_empty() { - return Ok(JavaSymbols(symbols)); - } - } - - Err(Error::invalid_value( - serde::de::Unexpected::Str(s.as_str()), - &"fully qualified symbol path: Class(SomeClassName)::ClassMethod(SomeClassMethod)", - )) - } -} - -pub trait Migrate { - fn make_migration(file: &Path) -> Result<()>; - fn make_example(file: &Path) -> Result<()>; -} - -impl Migrate for VulnComponentBatch { - fn make_example(out_file: &Path) -> Result<()> { - let content = indoc! {" - entries: - - cve: 'CVE-2022-37865' # ID of the CVE - dependency_revision_id: 'mvn+org.apache.ivy:ivy' # FOSSA locator: +$ - function: # Vulnerable Function - kind: java # Kind of function. Possible choices: 'java'. - symbol: 'Class(ZipPacking)::ClassMethod(unpack)' # Qualified Function Path. Possible choices: Class(), ClassMethod(), Constructor(). `::` denotes sub scope. - researcher: 'Name of the person' # Name of the researcher - notes: 'some notes' # Any notes (Optional) - "}; - - File::create(out_file) - .context("creating file")? - .write_all(content.as_bytes()) - .context("writing file") - .tap_ok(|_| println!("Wrote file at: {:?}", out_file)) - } - - fn make_migration(batch_file: &Path) -> Result<()> { - // 1. Read file - let file = - File::open(batch_file).with_context(|| format!("open file `{:?}`", batch_file))?; - let mut content = String::new(); - let mut reader = io::BufReader::new(file); - - reader - .read_to_string(&mut content) - .with_context(|| format!("read content of: `{:?}`", batch_file))?; - - let batch: VulnComponentBatch = - serde_yaml::from_str(&content).with_context(|| format!("parse: `{:?}`", batch_file))?; - - // 2. Create migration file's content - let entries: String = - serde_json::to_string_pretty(&batch.entries).expect("to serialize entries"); - let content = formatdoc! {" - // Automatically generated by: {app_name} {app_version} - /* eslint-disable quotes */ - /* eslint-disable quote-props */ - /* eslint-disable comma-dangle */ - const entries = {entries}; - - module.exports = {{ - up: async (queryInterface, _sequelize) => {{ - queryInterface.bulkInsert('ReachabilityPrivateVulnComponents', entries, {{ - updateOnDuplicate: ['evidence_notes', 'file_path', 'line_start', 'researcher', 'updated_at'], - upsertKeys: ['cve', 'dependency_revision_id', 'function'], - }}); - }}, - down: async (_queryInterface, _) => {{}}, - }}; - ", - app_name = crate_name!(), - app_version = crate_version!(), - }; - - // 3. Write migration file! - let out_file = &format!( - "{}-vulnComponent-automated.js", - Utc::now().format("%Y%m%d%H%M%S") - ); - File::create(out_file) - .context("creating file")? - .write_all(content.as_bytes()) - .context("writing file") - .tap_ok(|_| println!("Wrote file at: {:?}", out_file)) - } -} - -fn parse_symbol_java(input: &str) -> Option { - let mut parts = input.split("::"); - let kind_label_pair = parts.next()?; - let Some((kind, label)) = kind_label_pair.split_once('(') else { - return None; - }; - - let kind = kind.trim(); - let label = label.trim_end_matches(')'); - - match kind { - "Package" => Some(SymbolJava::Package { - label: label.to_string(), - }), - "Class" => Some(SymbolJava::Class { - label: label.to_string(), - }), - "ClassMethod" => Some(SymbolJava::ClassMethod { - label: label.to_string(), - }), - "Constructor" => Some(SymbolJava::Constructor { - label: label.to_string(), - }), - _ => None, - } -} - -/// Serializes `value` as into JSON string. -fn as_json_str(value: impl Serialize, serializer: S) -> Result -where - S: Serializer, -{ - serializer.serialize_str(&serde_json::to_string(&value).map_err(serde::ser::Error::custom)?) -} - -#[cfg(test)] -mod tests { - use crate::migration; - - use super::*; - use test_case::test_case; - - #[ - test_case("Class(Logger)::ClassMethod(log)", - vec![ - SymbolJava::Class {label: "Logger".to_string() }, - SymbolJava::ClassMethod { label: "log".to_string() } - ] - )] - #[ - test_case("Class(Logger)", - vec![ - SymbolJava::Class {label: "Logger".to_string() }, - ] - )] - fn parse_java_symbols_works(arg: &str, expected: Vec) { - let json_string = format!(r#"{{ "kind": "java", "symbol": "{arg}" }}"#); - let symbol: SymbolTarget = - serde_json::from_str(&json_string).expect("to serialize into json string"); - assert_eq!( - symbol, - SymbolTarget::Java { - symbol: migration::JavaSymbols(expected) - } - ) - } -} From fa086e54a148cfaf69177f3961a92ebda568c5bf Mon Sep 17 00:00:00 2001 From: Megh Date: Wed, 13 Dec 2023 10:14:28 -0700 Subject: [PATCH 9/9] use typedbuilder --- Cargo.lock | 23 ++++++++++++++++++++++- Cargo.toml | 1 + src/ingestion.rs | 39 +++++++++++++++++++-------------------- src/migration.rs | 19 ++++++++++--------- 4 files changed, 52 insertions(+), 30 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5b23a9a..9ce054f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -291,7 +291,7 @@ dependencies = [ "serde", "strum", "thiserror", - "typed-builder", + "typed-builder 0.10.0", ] [[package]] @@ -389,6 +389,7 @@ dependencies = [ "serde_yaml", "tap", "test-case", + "typed-builder 0.18.0", ] [[package]] @@ -596,6 +597,26 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "typed-builder" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e47c0496149861b7c95198088cbf36645016b1a0734cf350c50e2a38e070f38a" +dependencies = [ + "typed-builder-macro", +] + +[[package]] +name = "typed-builder-macro" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "982ee4197351b5c9782847ef5ec1fdcaf50503fb19d68f9771adae314e72b492" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.39", +] + [[package]] name = "unicode-ident" version = "1.0.12" diff --git a/Cargo.toml b/Cargo.toml index 7c087a6..c53dae6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,6 +18,7 @@ indoc = "2" locator = { git = "https://github.com/fossas/locator-rs.git", branch="main"} non-empty-string = { version="0.2.4", features= ["serde"] } tap = "1.0.1" +typed-builder = "0.18.0" [dev-dependencies] test-case = "3.3.1" diff --git a/src/ingestion.rs b/src/ingestion.rs index 31606cd..9c5c76b 100644 --- a/src/ingestion.rs +++ b/src/ingestion.rs @@ -2,6 +2,7 @@ use std::fs::File; use std::io::{self, Read, Write}; use std::path::Path; +use crate::migration; use anyhow::{Context, Result}; use chrono::Utc; use clap::{crate_name, crate_version}; @@ -12,8 +13,6 @@ use serde::de::Error; use serde::{Deserialize, Serialize}; use tap::TapFallible; -use crate::migration::MigrationVulnComponentEntry; - #[derive(Debug, Serialize, Deserialize)] pub struct VulnComponentBatch { pub entries: Vec, @@ -21,27 +20,27 @@ pub struct VulnComponentBatch { #[derive(Debug, Serialize, Deserialize)] pub struct VulnComponentEntry { - pub cve: NonEmptyString, - pub dependency_revision_id: Locator, - pub function: SymbolTarget, - pub researcher: NonEmptyString, - pub evidence_notes: Option, - pub file_path: Option, - pub line_start: Option, + cve: NonEmptyString, + dependency_revision_id: Locator, + function: SymbolTarget, + researcher: NonEmptyString, + evidence_notes: Option, + file_path: Option, + line_start: Option, } -impl TryFrom for MigrationVulnComponentEntry { +impl TryFrom for migration::VulnComponentEntry { type Error = serde_json::Error; fn try_from(v: VulnComponentEntry) -> Result { - Ok(Self { - cve: v.cve, - dependency_revision_id: v.dependency_revision_id, - function: serde_json::to_string(&v.function)?, - researcher: v.researcher, - evidence_notes: v.evidence_notes, - file_path: v.file_path, - line_start: v.line_start, - }) + Ok(migration::VulnComponentEntry::builder() + .cve(v.cve) + .dependency_revision_id(v.dependency_revision_id) + .function(serde_json::to_string(&v.function)?) + .researcher(v.researcher) + .evidence_notes(v.evidence_notes) + .file_path(v.file_path) + .line_start(v.line_start) + .build()) } } @@ -121,7 +120,7 @@ impl VulnComponentBatch { let entries = batch .entries .into_iter() - .map(MigrationVulnComponentEntry::try_from) + .map(migration::VulnComponentEntry::try_from) .collect::, _>>() .context("convert to output format")?; diff --git a/src/migration.rs b/src/migration.rs index 68655e6..7148149 100644 --- a/src/migration.rs +++ b/src/migration.rs @@ -1,14 +1,15 @@ use locator::Locator; use non_empty_string::NonEmptyString; use serde::{Deserialize, Serialize}; +use typed_builder::TypedBuilder; -#[derive(Debug, Serialize, Deserialize)] -pub struct MigrationVulnComponentEntry { - pub cve: NonEmptyString, - pub dependency_revision_id: Locator, - pub function: String, - pub researcher: NonEmptyString, - pub evidence_notes: Option, - pub file_path: Option, - pub line_start: Option, +#[derive(Debug, Serialize, Deserialize, TypedBuilder)] +pub struct VulnComponentEntry { + cve: NonEmptyString, + dependency_revision_id: Locator, + function: String, + researcher: NonEmptyString, + evidence_notes: Option, + file_path: Option, + line_start: Option, }