Ane 1060 broker smart imports (#134)

Broker smart imports

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "broker"
-version = "0.2.4-pre"
+version = "0.3.0-pre"
 edition = "2021"
 description = "The bridge between FOSSA and internal DevOps services"
 readme = "README.md"
@@ -70,6 +70,7 @@ tikv-jemallocator = { version = "0.5.4", optional = true }
 deadqueue = "0.2.4"
 governor = "0.6.0"
 nonzero_ext = "0.3.0"
+glob = "0.3.1"
 
 [dev-dependencies]
 insta = { version = "1.31.0", features = ["filters", "json", "yaml"] }

db/migrations/20230912062551_repo_state_2.down.sql

@@ -0,0 +1,2 @@
+-- Add down migration script here
+alter table repo_state drop column is_branch;

db/migrations/20230912062551_repo_state_2.up.sql

@@ -0,0 +1,2 @@
+-- Add up migration script here
+alter table repo_state add column is_branch integer;

docs/dev/README.md

@@ -165,3 +165,41 @@ git pull           # Ensure you're tagging the latest commit
 git tag v0.2.0     # Validate this is correct, and don't forget the `v`
 git push --tags    # Push the new tag to the remote.
 ```
+
+## smart imports
+
+Broker provides configurable branch/tag scanning for every integration. You can customize your scans 
+through these fields listed in the integrations section of your config.yml:
+
+```
+integrations:    
+  - type: git
+    import_branches: true  # Defaults to true
+    watched_branches:      # If unspecified, Broker will try to set to main or master if present
+      - main  
+      - release*             
+    import_tags: false     # Defaults to false
+```
+
+### default values
+
+If these fields are not set, `import_branches` will be set to `true`, `import_tags` will be set to `false`, and Broker 
+will make a best effort approach to set `watched_branches` to `main` or `master` if it is present in the remote.
+
+### branch scanning
+
+In order to scan specific branches, `import_branches` must be set to `true` and the list of branches you intend to scan should be provided under `watched_branches`. Having `watched_branches` set while having `import_branches` set to `false` is an invalid 
+combination and will cause Broker to throw errors. 
+
+[Glob matching](https://en.wikipedia.org/wiki/Glob_(programming)) is also provided with your branches. If one of your watched_branches is `release*` and your remote contains branches `release1`, `release2`, and `release-3`. Then all three 
+of those branches will be scanned due to glob matching.
+
+### tag scanning
+
+In order to allow Broker to scan tags in your remote, `import_tags` must be set to `true`
+
+### toggling fields
+
+Toggling `import_branches` from `true` to `false` will remove all existing uploaded scans for ALL branches of that particular remote in your local database (this does NOT delete your scans in the FOSSA UI). If toggled from `false` to `true`, Broker will perform as if it is scanning the listed `watched_branches` for the first time. On subsequent poll cycles, Broker will import the latest changes from your configured branches since the last revision (skipping any intermediate commits).
+
+Toggling `import_tags` from `true` to `false` will remove all existing uploaded scans for ALL tags of that particular remote in your local database (this does NOT delete your scans in the FOSSA UI). If toggled from `false` to `true`, Broker will perform as if it is scanning all the remote's tags for the first time. This would mean that all tags for that remote would be scanned. On subsequent poll cycles, Broker will import all created or changed tags since the last poll cycle.

docs/reference/config.md

@@ -69,11 +69,14 @@ This block specifies how to configure Broker to communicate with a git server fo
 
 | Value           | Required? | Description                                                                                   | Suggested default | Minimum value |
 |-----------------|-----------|-----------------------------------------------------------------------------------------------|-------------------|---------------|
-| `poll_interval` | Required  | How often Broker checks with the remote repository to see whether it has changed.<sup>1</sup> | `1 hour`          | `1 hour`      |
-| `remote`        | Required  | The remote git repository address.                                                            | N/A               | N/A           |
-| `auth`          | Required  | Required authentication to clone this repository.                                             | N/A               | N/A           |
-| `team`          | Optional  | The team in FOSSA to which this project should be assigned.<sup>2</sup>                       | N/A               | N/A           |
-| `title`         | Optional  | Specify a custom title for the project instead of using the default.<sup>3</sup>              | N/A               | N/A           |
+| `poll_interval`   | Required  | How often Broker checks with the remote repository to see whether it has changed.<sup>1</sup> | `1 hour`          | `1 hour`      |
+| `remote`          | Required  | The remote git repository address.                                                            | N/A               | N/A           |
+| `auth`            | Required  | Required authentication to clone this repository.                                             | N/A               | N/A           |
+| `team`            | Optional  | The team in FOSSA to which this project should be assigned.<sup>2</sup>                       | N/A               | N/A           |
+| `title`           | Optional  | Specify a custom title for the project instead of using the default.<sup>3</sup>              | N/A               | N/A           |
+| `import_branches` | Optional  | Initialize to scan specific branches for the remote repository                                | N/A               | N/A           |
+| `import_tags`     | Optional  | Initialize to scan tags for the remote repository                                             | N/A               | N/A           |
+| `watched_branches`| Optional  | The name of the branches that you intend to scan                                              | N/A               | N/A           |
 
 **[1]**: The poll interval defines the interval at which Broker _checks for updates_, not the interval at which Broker actually analyzes the repository.
 For more details on authentication, see [integration authentication](#integration-authentication).
@@ -118,6 +121,44 @@ Examples for valid durations:
 | `300ms 20s 5day`     | 5 days, 20 seconds, and 300 milliseconds |
 | `5day 4hours 10days` | 15 days and 4 hours                      |
 
+## Smart Imports
+
+Broker provides configurable branch/tag scanning for every integration. You can customize your scans 
+through these fields listed in the integrations section of your config.yml:
+
+```
+integrations:    
+  - type: git
+    import_branches: true  # Defaults to true
+    watched_branches:      # If unspecified, Broker will try to set to main or master if present
+      - main  
+      - release*             
+    import_tags: false     # Defaults to false
+```
+
+### default values
+
+If these fields are not set, `import_branches` will be set to `true`, `import_tags` will be set to `false`, and Broker 
+will make a best effort approach to set `watched_branches` to `main` or `master` if it is present in the remote.
+
+### branch scanning
+
+In order to scan specific branches, `import_branches` must be set to `true` and the list of branches you intend to scan should be provided under `watched_branches`. Having `watched_branches` set while having `import_branches` set to `false` is an invalid 
+combination and will cause Broker to throw errors. 
+
+[Glob matching](https://en.wikipedia.org/wiki/Glob_(programming)) is also provided with your branches. If one of your watched_branches is `release*` and your remote contains branches `release1`, `release2`, and `release-3`. Then all three 
+of those branches will be scanned due to glob matching.
+
+### tag scanning
+
+In order to allow Broker to scan tags in your remote, `import_tags` must be set to `true`
+
+### toggling fields
+
+Toggling `import_branches` from `true` to `false` will remove all existing uploaded scans for ALL branches of that particular remote in your local database (this does NOT delete your scans in the FOSSA UI). If toggled from `false` to `true`, Broker will perform as if it is scanning the listed `watched_branches` for the first time. On subsequent poll cycles, Broker will import the latest changes from your configured branches since the last revision (skipping any intermediate commits).
+
+Toggling `import_tags` from `true` to `false` will remove all existing uploaded scans for ALL tags of that particular remote in your local database (this does NOT delete your scans in the FOSSA UI). If toggled from `false` to `true`, Broker will perform as if it is scanning all the remote's tags for the first time. This would mean that all tags for that remote would be scanned. On subsequent poll cycles, Broker will import all created or changed tags since the last poll cycle.
+
 ## Integration authentication
 
 Integrations support several possible authentication schemes, specified by `type`.

src/api/remote.rs

@@ -17,6 +17,7 @@ use derive_more::{AsRef, Display, From};
 use derive_new::new;
 use error_stack::{ensure, report, Report, ResultExt};
 use getset::{CopyGetters, Getters};
+use glob::Pattern;
 use humantime::parse_duration;
 use serde::{Deserialize, Serialize};
 use tempfile::TempDir;
@@ -51,6 +52,14 @@ pub enum ValidationError {
     /// The provided value is empty.
     #[error("value is empty")]
     ValueEmpty,
+
+    /// Invalid combination of import branches and watched branches
+    #[error("validate import branches and watched branches")]
+    ImportBranches,
+
+    /// Unable to infer primary branch
+    #[error("primary branch could not be inferred")]
+    PrimaryBranch,
 }
 
 /// Validated config values for external code host integrations.
@@ -128,6 +137,18 @@ pub struct Integration {
     #[getset(get = "pub")]
     #[builder(setter(into))]
     protocol: Protocol,
+
+    /// Specifies if we want to scan specific branches
+    #[getset(get = "pub")]
+    import_branches: BranchImportStrategy,
+
+    /// Specifies if we want to scan tags
+    #[getset(get = "pub")]
+    import_tags: TagImportStrategy,
+
+    /// The name of the branches we want to scan
+    #[getset(get = "pub")]
+    watched_branches: Vec<WatchedBranch>,
 }
 
 impl Display for Integration {
@@ -151,6 +172,28 @@ impl Integration {
     pub fn endpoint(&self) -> &Remote {
         self.protocol().endpoint()
     }
+
+    /// Checks if the reference branch should be scanned by comparing it to our watched branches
+    pub fn should_scan_reference(&self, reference: &str) -> bool {
+        let branches = self.watched_branches();
+        for branch in branches {
+            match Pattern::new(branch.name()) {
+                Ok(p) => {
+                    if p.matches(reference) {
+                        return true;
+                    }
+                }
+                // In the case of error continue on and have the function return false if there are no matches
+                Err(_e) => continue,
+            }
+        }
+        false
+    }
+
+    /// Mutable reference for watched branches
+    pub fn add_watched_branch(&mut self, watched_branch: WatchedBranch) {
+        self.watched_branches.push(watched_branch)
+    }
 }
 
 /// Code is stored in many kinds of locations, from git repos to
@@ -214,6 +257,93 @@ impl TryFrom<String> for PollInterval {
     }
 }
 
+/// Specificies if we want to scan branches
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Display, Deserialize, Serialize, new)]
+pub enum BranchImportStrategy {
+    /// Scanning branches is not allowed
+    Disabled,
+    /// Scanning branches is allowed
+    Enabled,
+}
+
+impl From<Option<bool>> for BranchImportStrategy {
+    fn from(val: Option<bool>) -> BranchImportStrategy {
+        match val {
+            Some(false) => BranchImportStrategy::Disabled,
+            // True case maps to enabled and if it is None we default to enabled
+            _ => BranchImportStrategy::Enabled,
+        }
+    }
+}
+
+impl BranchImportStrategy {
+    /// Validates branch import configurations
+    pub fn is_valid(&self, watched_branches: &[WatchedBranch]) -> bool {
+        match self {
+            BranchImportStrategy::Disabled => watched_branches.is_empty(),
+            // On the Enabled variant, we can't check if watched branches is not empty here because Broker will try to infer watched branches
+            // Therefore just have the Enabled case map to true, and the validation on the enabled case will be checked later
+            BranchImportStrategy::Enabled => true,
+        }
+    }
+
+    /// Checks if we need to infer watched branches based on configuration
+    pub fn infer_watched_branches(&self, watched_branches: &[WatchedBranch]) -> bool {
+        match self {
+            BranchImportStrategy::Disabled => false,
+            BranchImportStrategy::Enabled => watched_branches.is_empty(),
+        }
+    }
+
+    /// Checks if scanning on branches should be skipped based on the enum variant
+    pub fn should_skip_branches(&self) -> bool {
+        match self {
+            BranchImportStrategy::Disabled => true,
+            BranchImportStrategy::Enabled => false,
+        }
+    }
+}
+
+/// Specifies if the we want to scan tags
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Display, Deserialize, Serialize, new)]
+pub enum TagImportStrategy {
+    /// Scanning tags is not allowed
+    Disabled,
+    /// Scanning tags is allowed
+    Enabled,
+}
+
+impl From<Option<bool>> for TagImportStrategy {
+    fn from(val: Option<bool>) -> TagImportStrategy {
+        match val {
+            Some(true) => TagImportStrategy::Enabled,
+            // False case maps to disabled, and if it is None we default to disabled
+            _ => TagImportStrategy::Disabled,
+        }
+    }
+}
+
+impl TagImportStrategy {
+    /// Checks if scanning tags should be skipped based on the enum variant
+    pub fn should_skip_tags(&self) -> bool {
+        match self {
+            TagImportStrategy::Disabled => true,
+            TagImportStrategy::Enabled => false,
+        }
+    }
+}
+
+/// The integration's branch that you intend to scan
+#[derive(Debug, Clone, PartialEq, Eq, AsRef, Display, Deserialize, Serialize, new)]
+pub struct WatchedBranch(String);
+
+impl WatchedBranch {
+    /// The name of the watched branch
+    pub fn name(&self) -> &str {
+        &self.0
+    }
+}
+
 /// Errors encountered while working with remotes
 #[derive(Debug, thiserror::Error)]
 pub enum RemoteProviderError {

src/api/remote/git.rs

@@ -5,6 +5,11 @@ use std::fmt::Display;
 use derive_new::new;
 use serde::{Deserialize, Serialize};
 
+/// Used to filter for main branch in an integration
+pub const MAIN_BRANCH: &str = "main";
+/// Used to filter for master branch in an integration
+pub const MASTER_BRANCH: &str = "master";
+
 /// A git reference's type (branch or tag)
 #[derive(Debug, Clone, Hash, Eq, PartialEq, Serialize, Deserialize, new)]
 pub enum Reference {

src/cmd/fix.rs

@@ -1,7 +1,10 @@
 //! Implementation for the fix command
 
 use crate::{
-    api::remote::{Reference, RemoteProvider, RemoteProviderError},
+    api::remote::{
+        git::{MAIN_BRANCH, MASTER_BRANCH},
+        Reference, RemoteProvider, RemoteProviderError,
+    },
     debug::{self, bundler, Bundle, BundleExport},
     ext::secrecy::REDACTION_LITERAL,
     fossa_cli::{self, DesiredVersion},
@@ -31,9 +34,6 @@ use crate::{
     ext::result::WrapErr,
 };
 
-const MAIN_BRANCH: &str = "main";
-const MASTER_BRANCH: &str = "master";
-
 /// Errors encountered when running the fix command.
 #[derive(Debug, thiserror::Error)]
 pub enum Error {