Merge pull request #293 from Interhyp/RELTEC-11838-pr-validation

Reltec 11838 pr validation

README.md

@@ -54,6 +54,8 @@ the [`local-config.yaml`][config] can be used to set the variables.
 |                                    |                                                       |                                                                                                                                                                                                                      |
 | `BITBUCKET_USERNAME`               |                                                       | Name of the user used for basic git authentication to pull and update the metadata repository.                                                                                                                       |
 | `BITBUCKET_PASSWORD`               |                                                       | Password of the user used for basic git authentication to pull and update the metadata repository.                                                                                                                   |
+| `PULL_REQUEST_BUILD_URL`           |                                                       | Url to link to on pull request builds. Should probably be the public URL of this service.                                                                                                                            |
+| `PULL_REQUEST_BUILD_KEY`           | `metadata-service`                                    | Key for pull request builds on pull requests in the underlying git repository. Changed files are syntactically validated.                                                                                            |
 |                                    |                                                       |                                                                                                                                                                                                                      |
 | `GIT_COMMITTER_NAME`               |                                                       | Name of the user used to create the Git commits.                                                                                                                                                                     |
 | `GIT_COMMITTER_EMAIL`              |                                                       | E-Mail of the user used to create the Git commits.                                                                                                                                                                   |

go.mod

@@ -51,6 +51,7 @@ require (
 	github.com/elastic/go-windows v1.0.1 // indirect
 	github.com/emirpasic/gods v1.18.1 // indirect
 	github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect
+	github.com/go-playground/webhooks/v6 v6.3.0 // indirect
 	github.com/goccy/go-json v0.10.2 // indirect
 	github.com/golang-jwt/jwt/v4 v4.5.0 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect

go.sum

@@ -94,9 +94,12 @@ github.com/go-git/go-git/v5 v5.12.0 h1:7Md+ndsjrzZxbddRDZjF14qK+NN56sy6wkqaVrjZt
 github.com/go-git/go-git/v5 v5.12.0/go.mod h1:FTM9VKtnI2m65hNI/TenDDDnUf2Q9FHnXYjuz9i5OEY=
 github.com/go-http-utils/headers v0.0.0-20181008091004-fed159eddc2a h1:v6zMvHuY9yue4+QkG/HQ/W67wvtQmWJ4SDo9aK/GIno=
 github.com/go-http-utils/headers v0.0.0-20181008091004-fed159eddc2a/go.mod h1:I79BieaU4fxrw4LMXby6q5OS9XnoR9UIKLOzDFjUmuw=
+github.com/go-playground/webhooks/v6 v6.3.0 h1:zBLUxK1Scxwi97TmZt5j/B/rLlard2zY7P77FHg58FE=
+github.com/go-playground/webhooks/v6 v6.3.0/go.mod h1:GCocmfMtpJdkEOM1uG9p2nXzg1kY5X/LtvQgtPHUaaA=
 github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU=
 github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
 github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
+github.com/gogits/go-gogs-client v0.0.0-20200905025246-8bb8a50cb355/go.mod h1:cY2AIrMgHm6oOHmR7jY+9TtjzSjQ3iG7tURJG3Y6XH0=
 github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg=
 github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0=
 github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE=

internal/acorn/config/customconfigint.go

@@ -33,6 +33,8 @@ type CustomConfiguration interface {
 
 	MetadataRepoUrl() string
 	MetadataRepoMainline() string
+	MetadataRepoProject() string
+	MetadataRepoName() string
 
 	UpdateJobIntervalCronPart() string
 	UpdateJobTimeoutSeconds() uint16
@@ -66,6 +68,9 @@ type CustomConfiguration interface {
 
 	RedisUrl() string
 	RedisPassword() string
+
+	PullRequestBuildUrl() string
+	PullRequestBuildKey() string
 }
 
 type NotificationConsumerConfig struct {
@@ -123,4 +128,6 @@ const (
 	KeyAllowedFileCategories          = "ALLOWED_FILE_CATEGORIES"
 	KeyRedisUrl                       = "REDIS_URL"
 	KeyRedisPassword                  = "REDIS_PASSWORD"
+	KeyPullRequestBuildUrl            = "PULL_REQUEST_BUILD_URL"
+	KeyPullRequestBuildKey            = "PULL_REQUEST_BUILD_KEY"
 )

internal/acorn/errors/httperror/error.go

@@ -8,6 +8,7 @@ import (
 type Error interface {
 	Ctx() context.Context
 	IsHttpError() bool
+	Status() int
 }
 
 // this also implements the error interface

internal/acorn/repository/bitbucketint.go

@@ -14,10 +14,23 @@ type Bitbucket interface {
 	GetBitbucketUser(ctx context.Context, username string) (BitbucketUser, error)
 	GetBitbucketUsers(ctx context.Context, usernames []string) ([]BitbucketUser, error)
 	FilterExistingUsernames(ctx context.Context, usernames []string) ([]string, error)
+
+	// GetChangedFilesOnPullRequest returns the file paths and contents list of changed files, and the
+	// head commit hash of the pull request source for which the files were obtained.
+	GetChangedFilesOnPullRequest(ctx context.Context, pullRequestId int) ([]File, string, error)
+
+	AddCommitBuildStatus(ctx context.Context, commitHash string, url string, key string, success bool) error
+
+	CreatePullRequestComment(ctx context.Context, pullRequestId int, comment string) error
 }
 
 type BitbucketUser struct {
 	Id     int    `json:"id"`
 	Name   string `json:"name"`
 	Active bool   `json:"active"`
 }
+
+type File struct {
+	Path     string
+	Contents string
+}

internal/acorn/service/prvalidatorint.go

@@ -0,0 +1,15 @@
+package service
+
+import "context"
+
+// PRValidator validates pull requests in the underlying repository to prevent bringing invalid content to the mainline.
+type PRValidator interface {
+	IsPRValidator() bool
+
+	// ValidatePullRequest validates the pull request, commenting on it and setting a build result.
+	//
+	// Failures to validate a pull request are not considered errors. Errors are only returned if
+	// the process of validation could not be completed (failure to respond by git server,
+	// could not obtain file list, etc.)
+	ValidatePullRequest(ctx context.Context, id uint64, toRef string, fromRef string) error
+}

internal/repository/bitbucket/bbclient/bbclient.go

@@ -140,10 +140,10 @@ func (c *Impl) betweenFailureAndRetry() aurestclientapi.BeforeRetryCallback {
 
 func (c *Impl) call(ctx context.Context, method string, requestUrlExtension string, requestBody interface{}, responseBodyPointer interface{}) error {
 	remoteUrl := fmt.Sprintf("%s/%s", c.apiBaseUrl, requestUrlExtension)
-	response := &aurestclientapi.ParsedResponse{
+	response := aurestclientapi.ParsedResponse{
 		Body: responseBodyPointer,
 	}
-	err := c.Client.Perform(ctx, method, remoteUrl, requestBody, response)
+	err := c.Client.Perform(ctx, method, remoteUrl, requestBody, &response)
 	if err != nil {
 		return err
 	}
@@ -167,3 +167,92 @@ func (c *Impl) GetBitbucketUser(ctx context.Context, username string) (repositor
 	err := c.call(ctx, http.MethodGet, urlExt, nil, &response)
 	return response, err
 }
+
+func (c *Impl) GetPullRequest(ctx context.Context, projectKey string, repositorySlug string, pullRequestId int32) (bbclientint.PullRequest, error) {
+	urlExt := fmt.Sprintf("%s/projects/%s/repos/%s/pull-requests/%d",
+		bbclientint.CoreApi,
+		url.PathEscape(projectKey),
+		url.PathEscape(repositorySlug),
+		pullRequestId)
+	response := bbclientint.PullRequest{}
+	err := c.call(ctx, http.MethodGet, urlExt, nil, &response)
+	return response, err
+}
+
+func (c *Impl) GetChanges(ctx context.Context, projectKey string, repositorySlug string, sinceHash string, untilHash string) (bbclientint.Changes, error) {
+	// since : main
+	// until : pr head
+	urlExt := fmt.Sprintf("%s/projects/%s/repos/%s/changes?since=%s&until=%s&limit=%d",
+		bbclientint.CoreApi,
+		url.PathEscape(projectKey),
+		url.PathEscape(repositorySlug),
+		url.QueryEscape(sinceHash),
+		url.QueryEscape(untilHash),
+		1000) // TODO pagination?
+	response := bbclientint.Changes{}
+	err := c.call(ctx, http.MethodGet, urlExt, nil, &response)
+	return response, err
+}
+
+func (c *Impl) getFileContentsPage(ctx context.Context, projectKey string, repositorySlug string, atHash string, path string, start int, limit int) (bbclientint.PaginatedLines, error) {
+	escapedPath := ""
+	for _, pathComponent := range strings.Split(path, "/") {
+		escapedPath += "/" + url.PathEscape(pathComponent)
+	}
+	urlExt := fmt.Sprintf("%s/projects/%s/repos/%s/browse/%s?at=%s&start=%d&limit=%d",
+		bbclientint.CoreApi,
+		url.PathEscape(projectKey),
+		url.PathEscape(repositorySlug),
+		escapedPath,
+		url.QueryEscape(atHash),
+		start,
+		limit)
+	response := bbclientint.PaginatedLines{}
+	err := c.call(ctx, http.MethodGet, urlExt, nil, &response)
+	return response, err
+}
+
+func (c *Impl) GetFileContentsAt(ctx context.Context, projectKey string, repositorySlug string, atHash string, path string) (string, error) {
+	var contents strings.Builder
+	var err error
+	start := 0
+
+	page := bbclientint.PaginatedLines{
+		IsLastPage:    false,
+		NextPageStart: &start,
+	}
+
+	for !page.IsLastPage && page.NextPageStart != nil {
+		page, err = c.getFileContentsPage(ctx, projectKey, repositorySlug, atHash, path, *page.NextPageStart, 1000)
+		if err != nil {
+			return contents.String(), err
+		}
+		for _, line := range page.Lines {
+			contents.WriteString(line.Text + "\n")
+		}
+	}
+
+	return contents.String(), nil
+}
+
+func (c *Impl) AddProjectRepositoryCommitBuildStatus(ctx context.Context, projectKey string, repositorySlug string, commitId string, commitBuildStatusRequest bbclientint.CommitBuildStatusRequest) error {
+	urlExt := fmt.Sprintf("%s/projects/%s/repos/%s/commits/%s/builds",
+		bbclientint.CoreApi,
+		url.PathEscape(projectKey),
+		url.PathEscape(repositorySlug),
+		url.PathEscape(commitId))
+
+	return c.call(ctx, http.MethodPost, urlExt, commitBuildStatusRequest, nil)
+}
+
+func (c *Impl) CreatePullRequestComment(ctx context.Context, projectKey string, repositorySlug string, pullRequestId int64, pullRequestCommentRequest bbclientint.PullRequestCommentRequest) (bbclientint.PullRequestComment, error) {
+	urlExt := fmt.Sprintf("%s/projects/%s/repos/%s/pull-requests/%d/comments",
+		bbclientint.CoreApi,
+		url.PathEscape(projectKey),
+		url.PathEscape(repositorySlug),
+		pullRequestId)
+
+	response := bbclientint.PullRequestComment{}
+	err := c.call(ctx, http.MethodPost, urlExt, pullRequestCommentRequest, &response)
+	return response, err
+}

internal/repository/bitbucket/bbclientint/bbclientint.go

@@ -9,6 +9,14 @@ type BitbucketClient interface {
 	Setup() error
 
 	GetBitbucketUser(ctx context.Context, username string) (repository.BitbucketUser, error)
+
+	GetPullRequest(ctx context.Context, projectKey string, repositorySlug string, pullRequestId int32) (PullRequest, error)
+	GetChanges(ctx context.Context, projectKey string, repositorySlug string, sinceHash string, untilHash string) (Changes, error)
+	GetFileContentsAt(ctx context.Context, projectKey string, repositorySlug string, atHash string, path string) (string, error)
+
+	AddProjectRepositoryCommitBuildStatus(ctx context.Context, projectKey string, repositorySlug string, commitId string, commitBuildStatusRequest CommitBuildStatusRequest) error
+
+	CreatePullRequestComment(ctx context.Context, projectKey string, repositorySlug string, pullRequestId int64, pullRequestCommentRequest PullRequestCommentRequest) (PullRequestComment, error)
 }
 
 const (