Restructuring and adding documentation

fossas · Sep 26, 2023 · bb2c2fb · bb2c2fb
1 parent 2a6f7ff
commit bb2c2fb
Show file tree

Hide file tree

Showing 11 changed files with 244 additions and 148 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -71,6 +71,7 @@ deadqueue = "0.2.4"
 governor = "0.6.0"
 nonzero_ext = "0.3.0"
 glob = "0.3.1"
+async-convert = "1.0.0"
 
 [dev-dependencies]
 insta = { version = "1.31.0", features = ["filters", "json", "yaml"] }

diff --git a/docs/dev/README.md b/docs/dev/README.md
@@ -165,3 +165,41 @@ git pull           # Ensure you're tagging the latest commit
 git tag v0.2.0     # Validate this is correct, and don't forget the `v`
 git push --tags    # Push the new tag to the remote.
 ```
+
+## smart imports
+
+Broker provides configurable branch/tag scanning for every integration. You can customize your scans 
+through these fields listed in the integrations section of your config.yml:
+
+```
+integrations:    
+  - type: git
+    import_branches: true  # Defaults to true
+    watched_branches:      # If unspecified, Broker will try to set to main or master if present
+      - main  
+      - release*             
+    import_tags: false     # Defaults to false
+```
+
+### default values
+
+If these fields are not set, `import_branches` will be set to `true`, `import_tags` will be set to `false`, and Broker 
+will make a best effort approach to set `watched_branches` to `main` or `master` if it is present in the remote.
+
+### branch scanning
+
+In order to scan specific branches, `import_branches` must be set to `true` and the list of branches you intend to scan should be provided under `watched_branches`. Having `watched_branches` set while having `import_branches` set to `false` is an invalid 
+combination and will cause Broker to throw errors. 
+
+[Glob matching](https://en.wikipedia.org/wiki/Glob_(programming)) is also provided with your branches. If one of your watched_branches is `release*` and your remote contains branches `release1`, `release2`, and `release-3`. Then all three 
+of those branches will be scanned due to glob matching.
+
+### tag scanning
+
+In order to allow Broker to scan tags in your remote, `import_tags` must be set to `true`
+
+### toggling fields
+
+Toggling `import_branches` from `true` to `false` will remove all existing uploaded scans for ALL branches of that particular remote in your database. If toggled from `false` to `true`, Broker will perform as if it is scanning the listed `watched_branches` for the first time.
+
+Toggling `import_tags` from `true` to `false` will remove all existing uploaded scans for ALL tags of that particular remote in your database. If toggled from `false` to `true`, Broker will perform as if it is scanning all the remote's tags for the first time. This would mean that all tags for that remote would be scanned.
diff --git a/docs/reference/config.md b/docs/reference/config.md
@@ -69,11 +69,14 @@ This block specifies how to configure Broker to communicate with a git server fo
 
 | Value           | Required? | Description                                                                                   | Suggested default | Minimum value |
 |-----------------|-----------|-----------------------------------------------------------------------------------------------|-------------------|---------------|
-| `poll_interval` | Required  | How often Broker checks with the remote repository to see whether it has changed.<sup>1</sup> | `1 hour`          | `1 hour`      |
-| `remote`        | Required  | The remote git repository address.                                                            | N/A               | N/A           |
-| `auth`          | Required  | Required authentication to clone this repository.                                             | N/A               | N/A           |
-| `team`          | Optional  | The team in FOSSA to which this project should be assigned.<sup>2</sup>                       | N/A               | N/A           |
-| `title`         | Optional  | Specify a custom title for the project instead of using the default.<sup>3</sup>              | N/A               | N/A           |
+| `poll_interval`   | Required  | How often Broker checks with the remote repository to see whether it has changed.<sup>1</sup> | `1 hour`          | `1 hour`      |
+| `remote`          | Required  | The remote git repository address.                                                            | N/A               | N/A           |
+| `auth`            | Required  | Required authentication to clone this repository.                                             | N/A               | N/A           |
+| `team`            | Optional  | The team in FOSSA to which this project should be assigned.<sup>2</sup>                       | N/A               | N/A           |
+| `title`           | Optional  | Specify a custom title for the project instead of using the default.<sup>3</sup>              | N/A               | N/A           |
+|`import_branches`  | Optional  | Initialize to scan specific branches for the remote repository                                | N/A               | N/A           |
+| `import_tags`     | Optional  | Initialize to scan tags for the remote repository                                             | N/A               | N/A           |
+| `watched_branches`| Optional  | The name of the branches that you intend to scan                                              | N/A               | N/A           |
 
 **[1]**: The poll interval defines the interval at which Broker _checks for updates_, not the interval at which Broker actually analyzes the repository.
 For more details on authentication, see [integration authentication](#integration-authentication).
@@ -118,6 +121,44 @@ Examples for valid durations:
 | `300ms 20s 5day`     | 5 days, 20 seconds, and 300 milliseconds |
 | `5day 4hours 10days` | 15 days and 4 hours                      |
 
+## Smart Imports
+
+Broker provides configurable branch/tag scanning for every integration. You can customize your scans 
+through these fields listed in the integrations section of your config.yml:
+
+```
+integrations:    
+  - type: git
+    import_branches: true  # Defaults to true
+    watched_branches:      # If unspecified, Broker will try to set to main or master if present
+      - main  
+      - release*             
+    import_tags: false     # Defaults to false
+```
+
+### default values
+
+If these fields are not set, `import_branches` will be set to `true`, `import_tags` will be set to `false`, and Broker 
+will make a best effort approach to set `watched_branches` to `main` or `master` if it is present in the remote.
+
+### branch scanning
+
+In order to scan specific branches, `import_branches` must be set to `true` and the list of branches you intend to scan should be provided under `watched_branches`. Having `watched_branches` set while having `import_branches` set to `false` is an invalid 
+combination and will cause Broker to throw errors. 
+
+[Glob matching](https://en.wikipedia.org/wiki/Glob_(programming)) is also provided with your branches. If one of your watched_branches is `release*` and your remote contains branches `release1`, `release2`, and `release-3`. Then all three 
+of those branches will be scanned due to glob matching.
+
+### tag scanning
+
+In order to allow Broker to scan tags in your remote, `import_tags` must be set to `true`
+
+### toggling fields
+
+Toggling `import_branches` from `true` to `false` will remove all existing uploaded scans for ALL branches of that particular remote in your database. If toggled from `false` to `true`, Broker will perform as if it is scanning the listed `watched_branches` for the first time.
+
+Toggling `import_tags` from `true` to `false` will remove all existing uploaded scans for ALL tags of that particular remote in your database. If toggled from `false` to `true`, Broker will perform as if it is scanning all the remote's tags for the first time. This would mean that all tags for that remote would be scanned.
+
 ## Integration authentication
 
 Integrations support several possible authentication schemes, specified by `type`.

diff --git a/src/api/remote.rs b/src/api/remote.rs
@@ -21,7 +21,6 @@ use glob::Pattern;
 use humantime::parse_duration;
 use serde::{Deserialize, Serialize};
 use tempfile::TempDir;
-use tracing::warn;
 use typed_builder::TypedBuilder;
 
 use crate::{
@@ -32,11 +31,6 @@ use crate::{
     },
 };
 
-/// Used to filter for main branch in an integration
-pub const MAIN_BRANCH: &str = "main";
-/// Used to filter for master branch in an integration
-pub const MASTER_BRANCH: &str = "master";
-
 /// Integrations for git repositories
 pub mod git;
 
@@ -63,8 +57,8 @@ pub enum ValidationError {
     #[error("validate import branches and watched branches")]
     ImportBranches,
 
-    /// Unable to decipher primary branch
-    #[error("primary branch could not be deciphered")]
+    /// Unable to infer primary branch
+    #[error("primary branch could not be inferred")]
     PrimaryBranch,
 }
 
@@ -146,11 +140,11 @@ pub struct Integration {
 
     /// Specifies if we want to scan specific branches
     #[getset(get = "pub")]
-    import_branches: bool,
+    import_branches: BranchImportStrategy,
 
-    /// Specifies if we want to scan specific tags
+    /// Specifies if we want to scan tags
     #[getset(get = "pub")]
-    import_tags: bool,
+    import_tags: TagImportStrategy,
 
     /// The name of the branches we want to scan
     #[getset(get = "pub")]
@@ -179,61 +173,11 @@ impl Integration {
         self.protocol().endpoint()
     }
 
-    /// Best effort approach to set primary branch by searching for main/master branch
-    pub async fn set_watched_branches(&self) -> Result<Integration, Report<ValidationError>> {
-        match self {
-            Integration {
-                poll_interval,
-                team,
-                title,
-                protocol,
-                import_branches: true,
-                import_tags,
-                watched_branches,
-            } => {
-                if !watched_branches.is_empty() {
-                    return Ok(self.clone());
-                }
-
-                let references = self.references().await.unwrap_or_default();
-                let primary_branch = references
-                    .iter()
-                    .find(|r| r.name() == MAIN_BRANCH || r.name() == MASTER_BRANCH)
-                    .cloned();
-                match primary_branch {
-                    None => {
-                        report!(ValidationError::PrimaryBranch)
-                        .wrap_err()
-                        .help("watched_branches was empty and failed to inject main/master branch into watched_branches")
-                        .describe_lazy(||"provide valid watched_branches")?
-                    }
-                    Some(branch) => {
-                        let primary_branch_name = branch.name();
-                        warn!("Watched_branches was set to empty, added branch '{primary_branch_name}' as a best effort approach");
-                        let watched_branch = WatchedBranch::new(branch.name().to_string());
-                        let watched_branches = vec![watched_branch];
-
-                        Integration {
-                            poll_interval: *poll_interval,
-                            team: team.clone(),
-                            title: title.clone(),
-                            protocol: protocol.clone(),
-                            import_branches: true,
-                            import_tags: *import_tags,
-                            watched_branches,
-                        }.wrap_ok()
-                    }
-                }
-            }
-            _ => Ok(self.clone()),
-        }
-    }
-
     /// Checks if the reference branch should be scanned by comparing it to our watched branches
-    pub fn validate_reference_scan(&self, reference: &str) -> bool {
+    pub fn should_scan_reference(&self, reference: &str) -> bool {
         let branches = self.watched_branches();
         for branch in branches {
-            match Pattern::new(branch.name().as_str()) {
+            match Pattern::new(branch.name()) {
                 Ok(p) => {
                     if p.matches(reference) {
                         return true;
@@ -313,14 +257,52 @@ impl TryFrom<String> for PollInterval {
     }
 }
 
+/// Specificies if we want to scan branches
+#[derive(Debug, Clone, PartialEq, Eq, Display, Deserialize, Serialize, new)]
+pub enum BranchImportStrategy {
+    /// Do not scan integration branches
+    Disabled,
+    /// Scan integration branches
+    Enabled,
+}
+
+impl From<Option<bool>> for BranchImportStrategy {
+    fn from(val: Option<bool>) -> BranchImportStrategy {
+        match val {
+            Some(false) => BranchImportStrategy::Disabled,
+            // True case maps to enabled and if it is None we default to enabled
+            _ => BranchImportStrategy::Enabled,
+        }
+    }
+}
+
+/// Specifies if the we want to scan tags
+#[derive(Debug, Clone, PartialEq, Eq, Display, Deserialize, Serialize, new)]
+pub enum TagImportStrategy {
+    /// Do not scan integration tags
+    Disabled,
+    /// Scan integration tags
+    Enabled,
+}
+
+impl From<Option<bool>> for TagImportStrategy {
+    fn from(val: Option<bool>) -> TagImportStrategy {
+        match val {
+            Some(true) => TagImportStrategy::Enabled,
+            // False case maps to disabled, and if it is None we default to disabled
+            _ => TagImportStrategy::Disabled,
+        }
+    }
+}
+
 /// The integration's branch that you intend to scan
 #[derive(Debug, Clone, PartialEq, Eq, AsRef, Display, Deserialize, Serialize, new)]
 pub struct WatchedBranch(String);
 
 impl WatchedBranch {
     /// The name of the watched branch
-    pub fn name(&self) -> String {
-        self.0.clone()
+    pub fn name(&self) -> &str {
+        &self.0
     }
 }
 
@@ -364,13 +346,6 @@ impl Reference {
             Reference::Git(git) => git.name().as_str(),
         }
     }
-
-    /// The reference's type (branch/tag)
-    pub fn reference_type(&self) -> &git::Reference {
-        match self {
-            Reference::Git(reference) => reference,
-        }
-    }
 }
 
 impl Display for Reference {

diff --git a/src/api/remote/git.rs b/src/api/remote/git.rs
@@ -5,6 +5,11 @@ use std::fmt::Display;
 use derive_new::new;
 use serde::{Deserialize, Serialize};
 
+/// Used to filter for main branch in an integration
+pub const MAIN_BRANCH: &str = "main";
+/// Used to filter for master branch in an integration
+pub const MASTER_BRANCH: &str = "master";
+
 /// A git reference's type (branch or tag)
 #[derive(Debug, Clone, Hash, Eq, PartialEq, Serialize, Deserialize, new)]
 pub enum Reference {

diff --git a/src/cmd/fix.rs b/src/cmd/fix.rs
@@ -1,7 +1,10 @@
 //! Implementation for the fix command
 
 use crate::{
-    api::remote::{Reference, RemoteProvider, RemoteProviderError, MAIN_BRANCH, MASTER_BRANCH},
+    api::remote::{
+        git::{MAIN_BRANCH, MASTER_BRANCH},
+        Reference, RemoteProvider, RemoteProviderError,
+    },
     debug::{self, bundler, Bundle, BundleExport},
     ext::secrecy::REDACTION_LITERAL,
     fossa_cli::{self, DesiredVersion},