From 976446a5d0bcf759442aaba8c65ae92527eae846 Mon Sep 17 00:00:00 2001 From: Aaron Lun Date: Wed, 10 Apr 2024 09:59:15 -0700 Subject: [PATCH] Replace fsnotify with a HTTP request to trigger actions. (#11) Most shared filesystems are networked in some manner and don't share events across nodes; this means that fsnotify doesn't actually work. Polling is too taxing so instead we ask the client to ping an API to indicate that a request body has been written to the staging directory and is ready for execution. We don't ask the client to provide the request details in the API as it's unauthenticated. We're still relying on the Unix file owner to tell us who is making the request; the ping just tells us that the file is ready, which happily eliminates the need for the retry loop. The output is also returned as a HTTP response, which eliminates the need for the responses directory. Some extra work is involved in making sure that the correct HTTP status codes are reported for the different errors. We also mandate go >= 1.22.1 now. --- .github/workflows/build.yaml | 2 +- README.md | 39 +++--- create.go | 15 +- delete.go | 31 +++-- go.mod | 7 +- go.sum | 4 - latest.go | 11 +- main.go | 259 ++++++++++++++--------------------- permissions.go | 11 +- probation.go | 13 +- purge.go | 11 +- purge_test.go | 30 ++-- upload.go | 25 ++-- usage.go | 9 +- utils.go | 21 +-- utils_test.go | 28 ---- 16 files changed, 207 insertions(+), 309 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index a012b38..dc41e24 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -24,7 +24,7 @@ jobs: - name: Setup Go uses: actions/setup-go@v4 with: - go-version: '1.20' + go-version: '1.22' cache-dependency-path: go.sum - name: Install dependencies diff --git a/README.md b/README.md index 32f2c59..c4a20cf 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,6 @@ The Gobbler implements [**gypsum**](https://github.com/ArtifactDB/gypsum-worker)-like storage of ArtifactDB-managed files on a shared filesystem. This replaces cloud storage with a world-readable local directory, reducing costs and improving efficiency by avoiding network traffic for uploads/downloads. We simplify authentication by using Unix file permissions to determine ownership, avoiding the need for a separate identity provider like GitHub. -In fact, no HTTP requests are required at all, as the user and Gobbler communicate solely through filesystem events. This document is intended for system administrators who want to spin up their own instance or developers of new clients to the Gobbler service. Users should never have to interact with the Gobbler directly, as this should be mediated by client packages in relevant frameworks like R or Python. @@ -135,22 +134,13 @@ Each project's current usage is tracked in `{project}/..usage`, which contains a ### General instructions -Gobbler uses filesystem events to watch a "staging directory", a world-writeable directory on the shared filesystem. -Users submit requests to the Gobbler by simply writing a JSON file with the request parameters inside the staging directory. +The Gobbler requires a "staging directory", a world-writeable directory on the shared filesystem. +Users submit requests to the Gobbler by writing a JSON file with the request parameters inside the staging directory. Each request file's name should have a prefix of `request--` where `ACTION` specifies the action to be performed. -Upon creation of a request file, the Gobbler will parse it and execute the request with the specified parameters. - -After completing the request, the Gobbler will write a JSON response to the `responses` subdirectory of the staging directory. -This has the same name as the initial request file, so users can easily poll the subdirectory for the existence of this file. -Each response will have at least the `status` property (either `SUCCESS` or `FAILED`). +Once this file is written, users should perform a POST request to the Gobbler API to trigger execution; +this will return a JSON response that has at least the `status` property (either `SUCCESS` or `FAILED`). For failures, this will be an additional `reason` string property to specify the reason; -for successes, additional proeprties may be present depending on the request action. - -When writing the request file, it is recommended to use the write-and-rename paradigm. -Specifically, users should write the JSON request body to a file inside the staging directory that does _not_ have the `request--` prefix. -Once the write is complete, this file can be renamed to a file with said prefix. -This ensures that the Gobbler does not read a partially-written file. -(That said, a direct write to the final file can still be performed, in which case the Gobbler will perform a few retries to avoid errors from parsing an incomplete file.) +for successes, additional properties may be present depending on the request action. ### Creating projects (admin) @@ -318,27 +308,30 @@ cd gobbler && go build ``` Then, set up a staging directory with global read/write permissions. - -- The staging directory should be on a filesystem supported by the [`fsnotify`](httsp://github.com/fsnotify/fsnotify) package. -- All parent directories of the staging directory should be at least globally executable. +ll parent directories of the staging directory should be at least globally executable. ```sh mkdir STAGING chmod 777 STAGING ``` -Then, set up a registry directory with global read-only permissions. - -- The registry and staging directories do not need to be on the same filesystem (e.g., for mounted shares), as long as both are accessible to users. +Next, set up a registry directory with global read-only permissions. +Note that the registry and staging directories do not need to be on the same filesystem (e.g., for mounted shares), as long as both are accessible to users. ```sh mkdir REGISTRY chmod 755 REGISTRY ``` -The Gobbler can then be started by running the binary with a few arguments, including the UIDs of administrators: +Finally, start the Gobbler by running the binary with a few arguments, including the UIDs of administrators: ```sh -./gobbler -staging STAGING -registry REGISTRY -admin ADMIN1,ADMIN2 +./gobbler \ + -staging STAGING \ + -registry REGISTRY \ + -admin ADMIN1,ADMIN2 \ + -port PORT ``` +For requests, clients should write to `STAGING` and hit the API at `PORT` (or any equivalent alias). +All registered files can be read from `REGISTRY`. diff --git a/create.go b/create.go index 005b23d..6adaec8 100644 --- a/create.go +++ b/create.go @@ -8,6 +8,7 @@ import ( "encoding/json" "strconv" "errors" + "net/http" ) func createProjectHandler(reqpath string, globals *globalConfiguration) error { @@ -16,7 +17,7 @@ func createProjectHandler(reqpath string, globals *globalConfiguration) error { return fmt.Errorf("failed to find owner of %q; %w", reqpath, err) } if !isAuthorizedToAdmin(req_user, globals.Administrators) { - return fmt.Errorf("user %q is not authorized to create a project", req_user) + return newHttpError(http.StatusForbidden, fmt.Errorf("user %q is not authorized to create a project", req_user)) } request := struct { @@ -27,15 +28,15 @@ func createProjectHandler(reqpath string, globals *globalConfiguration) error { // Reading in the request. handle, err := os.ReadFile(reqpath) if err != nil { - return &readRequestError{ Cause: fmt.Errorf("failed to read %q; %w", reqpath, err) } + return fmt.Errorf("failed to read %q; %w", reqpath, err) } err = json.Unmarshal(handle, &request) if err != nil { - return &readRequestError{ Cause: fmt.Errorf("failed to parse JSON from %q; %w", reqpath, err) } + return newHttpError(http.StatusBadRequest, fmt.Errorf("failed to parse JSON from %q; %w", reqpath, err)) } if request.Project == nil { - return &readRequestError{ Cause: fmt.Errorf("expected a 'project' property in %q", reqpath) } + return newHttpError(http.StatusBadRequest, fmt.Errorf("expected a 'project' property in %q", reqpath)) } project := *(request.Project) @@ -45,13 +46,13 @@ func createProjectHandler(reqpath string, globals *globalConfiguration) error { func createProject(project string, inperms *unsafePermissionsMetadata, req_user string, globals *globalConfiguration) error { err := isBadName(project) if err != nil { - return fmt.Errorf("invalid project name; %w", err) + return newHttpError(http.StatusBadRequest, fmt.Errorf("invalid project name; %w", err)) } // Creating a new project from a pre-supplied name. project_dir := filepath.Join(globals.Registry, project) if _, err = os.Stat(project_dir); !errors.Is(err, os.ErrNotExist) { - return fmt.Errorf("project %q already exists", project) + return newHttpError(http.StatusBadRequest, fmt.Errorf("project %q already exists", project)) } // No need to lock before MkdirAll, it just no-ops if the directory already exists. @@ -72,7 +73,7 @@ func createProject(project string, inperms *unsafePermissionsMetadata, req_user if inperms != nil && inperms.Uploaders != nil { san, err := sanitizeUploaders(inperms.Uploaders) if err != nil { - return fmt.Errorf("invalid 'permissions.uploaders' in the request details; %w", err) + return newHttpError(http.StatusBadRequest, fmt.Errorf("invalid 'permissions.uploaders' in the request details; %w", err)) } perms.Uploaders = san } else { diff --git a/delete.go b/delete.go index 9b3d66e..285351f 100644 --- a/delete.go +++ b/delete.go @@ -7,6 +7,7 @@ import ( "path/filepath" "time" "errors" + "net/http" ) func deleteProjectHandler(reqpath string, globals *globalConfiguration) error { @@ -15,7 +16,7 @@ func deleteProjectHandler(reqpath string, globals *globalConfiguration) error { return fmt.Errorf("failed to find owner of %q; %w", reqpath, err) } if !isAuthorizedToAdmin(req_user, globals.Administrators) { - return fmt.Errorf("user %q is not authorized to delete a project", req_user) + return newHttpError(http.StatusForbidden, fmt.Errorf("user %q is not authorized to delete a project", req_user)) } incoming := struct { @@ -24,17 +25,17 @@ func deleteProjectHandler(reqpath string, globals *globalConfiguration) error { { handle, err := os.ReadFile(reqpath) if err != nil { - return &readRequestError{ Cause: fmt.Errorf("failed to read %q; %w", reqpath, err) } + return fmt.Errorf("failed to read %q; %w", reqpath, err) } err = json.Unmarshal(handle, &incoming) if err != nil { - return &readRequestError{ Cause: fmt.Errorf("failed to parse JSON from %q; %w", reqpath, err) } + return newHttpError(http.StatusBadRequest, fmt.Errorf("failed to parse JSON from %q; %w", reqpath, err)) } err = isMissingOrBadName(incoming.Project) if err != nil { - return fmt.Errorf("invalid 'project' property in %q; %w", reqpath, err) + return newHttpError(http.StatusBadRequest, fmt.Errorf("invalid 'project' property in %q; %w", reqpath, err)) } } @@ -65,7 +66,7 @@ func deleteAssetHandler(reqpath string, globals *globalConfiguration) error { return fmt.Errorf("failed to find owner of %q; %w", reqpath, err) } if !isAuthorizedToAdmin(req_user, globals.Administrators) { - return fmt.Errorf("user %q is not authorized to delete a project", req_user) + return newHttpError(http.StatusForbidden, fmt.Errorf("user %q is not authorized to delete a project", req_user)) } incoming := struct { @@ -75,22 +76,22 @@ func deleteAssetHandler(reqpath string, globals *globalConfiguration) error { { handle, err := os.ReadFile(reqpath) if err != nil { - return &readRequestError{ Cause: fmt.Errorf("failed to read %q; %w", reqpath, err) } + return fmt.Errorf("failed to read %q; %w", reqpath, err) } err = json.Unmarshal(handle, &incoming) if err != nil { - return &readRequestError{ Cause: fmt.Errorf("failed to parse JSON from %q; %w", reqpath, err) } + return newHttpError(http.StatusBadRequest, fmt.Errorf("failed to parse JSON from %q; %w", reqpath, err)) } err = isMissingOrBadName(incoming.Project) if err != nil { - return fmt.Errorf("invalid 'project' property in %q; %w", reqpath, err) + return newHttpError(http.StatusBadRequest, fmt.Errorf("invalid 'project' property in %q; %w", reqpath, err)) } err = isMissingOrBadName(incoming.Asset) if err != nil { - return fmt.Errorf("invalid 'asset' property in %q; %w", reqpath, err) + return newHttpError(http.StatusBadRequest, fmt.Errorf("invalid 'asset' property in %q; %w", reqpath, err)) } } @@ -150,7 +151,7 @@ func deleteVersionHandler(reqpath string, globals *globalConfiguration) error { return fmt.Errorf("failed to find owner of %q; %w", reqpath, err) } if !isAuthorizedToAdmin(req_user, globals.Administrators) { - return fmt.Errorf("user %q is not authorized to delete a project", req_user) + return newHttpError(http.StatusForbidden, fmt.Errorf("user %q is not authorized to delete a project", req_user)) } incoming := struct { @@ -161,25 +162,25 @@ func deleteVersionHandler(reqpath string, globals *globalConfiguration) error { { handle, err := os.ReadFile(reqpath) if err != nil { - return &readRequestError{ Cause: fmt.Errorf("failed to read %q; %w", reqpath, err) } + return fmt.Errorf("failed to read %q; %w", reqpath, err) } err = json.Unmarshal(handle, &incoming) if err != nil { - return &readRequestError{ Cause: fmt.Errorf("failed to parse JSON from %q; %w", reqpath, err) } + return newHttpError(http.StatusBadRequest, fmt.Errorf("failed to parse JSON from %q; %w", reqpath, err)) } err = isMissingOrBadName(incoming.Project) if err != nil { - return fmt.Errorf("invalid 'project' property in %q; %w", reqpath, err) + return newHttpError(http.StatusBadRequest, fmt.Errorf("invalid 'project' property in %q; %w", reqpath, err)) } err = isMissingOrBadName(incoming.Asset) if err != nil { - return fmt.Errorf("invalid 'asset' property in %q; %w", reqpath, err) + return newHttpError(http.StatusBadRequest, fmt.Errorf("invalid 'asset' property in %q; %w", reqpath, err)) } err = isMissingOrBadName(incoming.Version) if err != nil { - return fmt.Errorf("invalid 'version' property in %q; %w", reqpath, err) + return newHttpError(http.StatusBadRequest, fmt.Errorf("invalid 'version' property in %q; %w", reqpath, err)) } } diff --git a/go.mod b/go.mod index 01bd050..f8288be 100644 --- a/go.mod +++ b/go.mod @@ -1,8 +1,3 @@ module gobbler -go 1.20 - -require ( - github.com/fsnotify/fsnotify v1.7.0 // indirect - golang.org/x/sys v0.4.0 // indirect -) +go 1.22.1 diff --git a/go.sum b/go.sum index ccd7ce9..e69de29 100644 --- a/go.sum +++ b/go.sum @@ -1,4 +0,0 @@ -github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= -github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= -golang.org/x/sys v0.4.0 h1:Zr2JFtRQNX3BCZ8YtxRE9hNJYC8J6I1MVbMg6owUp18= -golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/latest.go b/latest.go index d4d7fd9..54d3296 100644 --- a/latest.go +++ b/latest.go @@ -7,6 +7,7 @@ import ( "path/filepath" "time" "strings" + "net/http" ) type latestMetadata struct { @@ -90,7 +91,7 @@ func refreshLatestHandler(reqpath string, globals *globalConfiguration) (*latest } if !isAuthorizedToAdmin(source_user, globals.Administrators) { - return nil, fmt.Errorf("user %q is not authorized to refresh the latest version (%q)", source_user, reqpath) + return nil, newHttpError(http.StatusForbidden, fmt.Errorf("user %q is not authorized to refresh the latest version (%q)", source_user, reqpath)) } incoming := struct { @@ -100,22 +101,22 @@ func refreshLatestHandler(reqpath string, globals *globalConfiguration) (*latest { handle, err := os.ReadFile(reqpath) if err != nil { - return nil, &readRequestError{ fmt.Errorf("failed to read %q; %w", reqpath, err) } + return nil, fmt.Errorf("failed to read %q; %w", reqpath, err) } err = json.Unmarshal(handle, &incoming) if err != nil { - return nil, &readRequestError{ fmt.Errorf("failed to parse JSON from %q; %w", reqpath, err) } + return nil, newHttpError(http.StatusBadRequest, fmt.Errorf("failed to parse JSON from %q; %w", reqpath, err)) } err = isMissingOrBadName(incoming.Project) if err != nil { - return nil, fmt.Errorf("invalid 'project' property in %q; %w", reqpath, err) + return nil, newHttpError(http.StatusBadRequest, fmt.Errorf("invalid 'project' property in %q; %w", reqpath, err)) } err = isMissingOrBadName(incoming.Asset) if err != nil { - return nil, fmt.Errorf("invalid 'asset' property in %q; %w", reqpath, err) + return nil, newHttpError(http.StatusBadRequest, fmt.Errorf("invalid 'asset' property in %q; %w", reqpath, err)) } } diff --git a/main.go b/main.go index bb9fc9e..aa1f74c 100644 --- a/main.go +++ b/main.go @@ -2,7 +2,6 @@ package main import ( "log" - "github.com/fsnotify/fsnotify" "flag" "path/filepath" "time" @@ -10,12 +9,38 @@ import ( "errors" "strings" "fmt" + "encoding/json" + "net/http" + "strconv" ) +func dumpJsonResponse(w http.ResponseWriter, status int, v interface{}, path string) { + contents, err := json.Marshal(v) + if err != nil { + log.Printf("failed to convert response to JSON for %q; %v", path, err) + contents = []byte("unknown") + } + + w.Header().Set("Content-Type", "application/json") + w.Header().Set("Access-Control-Allow-Origin", "*") + w.WriteHeader(status) + _, err = w.Write(contents) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + log.Printf("failed to write JSON response for %q; %v", path, err) + } +} + +func dumpErrorResponse(w http.ResponseWriter, status int, message string, path string) { + log.Printf("failed to process %q; %s\n", path, message) + dumpJsonResponse(w, status, map[string]interface{}{ "status": "ERROR", "reason": message }, path) +} + func main() { - spath := flag.String("staging", "", "Path to the staging directory to be watched") - rpath := flag.String("registry", "", "Path to the registry") + spath := flag.String("staging", "", "Path to the staging directory.") + rpath := flag.String("registry", "", "Path to the registry.") mstr := flag.String("admin", "", "Comma-separated list of administrators.") + port := flag.Int("port", 8080, "Port to listen to API requests.") flag.Parse() if *spath == "" || *rpath == "" { @@ -29,21 +54,6 @@ func main() { globals.Administrators = strings.Split(*mstr, ",") } - // Setting up special subdirectories. - response_name := "responses" - response_dir := filepath.Join(staging, response_name) - if _, err := os.Stat(response_dir); errors.Is(err, os.ErrNotExist) { - err := os.Mkdir(response_dir, 0755) - if err != nil { - log.Fatalf("failed to create a 'responses' subdirectory in %q; %v", staging, err) - } - } else { - err := os.Chmod(response_dir, 0755) - if err != nil { - log.Fatalf("failed to validate permissions for the 'responses' subdirectory in %q; %v", staging, err) - } - } - log_dir := filepath.Join(globals.Registry, logDirName) if _, err := os.Stat(log_dir); errors.Is(err, os.ErrNotExist) { err := os.Mkdir(log_dir, 0755) @@ -53,165 +63,104 @@ func main() { } // Launching a watcher to pick up changes and launch jobs. - watcher, err := fsnotify.NewWatcher() - if err != nil { - log.Fatal("failed to start the watcher on the staging directory; ", err) - } - defer watcher.Close() + http.HandleFunc("POST /new/{path}", func(w http.ResponseWriter, r *http.Request) { + path := filepath.Base(r.PathValue("path")) + log.Println("processing " + path) - go func() { - for { - select { - case event, ok := <-watcher.Events: - if !ok { - return - } - log.Println("triggered filesystem event:", event) - - // It is recommended that request bodies should be initially - // written to some other file (e.g., `.tmpXXXX`) inside the - // staging directory, and then moved to the actual file name - // (`request--YYY`). The rename should be atomic and - // thus we avoid problems with the code below triggering before - // the requester has completed the write of the body. Under - // this logic, we only have to watch the Create events as no - // Writes are being performed on a renamed file. - if event.Has(fsnotify.Create) { - info, err := os.Stat(event.Name) - if errors.Is(err, os.ErrNotExist) { - continue - } else if err != nil { - log.Println("failed to stat;", err) - continue - } - - if info.IsDir() { - continue - } - - basename := filepath.Base(event.Name) - if strings.HasPrefix(basename, "request-") { - reqtype := strings.TrimPrefix(basename, "request-") - - go func(reqpath, basename string) { - var reportable_err error - payload := map[string]interface{}{} - - // We prefer an atomic write, but nonetheless, if a request is directly written to the final - // file name, we will continuously retry (for up to 1 second) if there is any error during - // the reading of the request body. This takes advantage of the fact that an incompletely - // written JSON object must be invalid as it's missing the closing brace. - for i := 0; i < 4; i++ { - if strings.HasPrefix(reqtype, "upload-") { - reportable_err = uploadHandler(reqpath, &globals) - - } else if strings.HasPrefix(reqtype, "refresh_latest-") { - res, err0 := refreshLatestHandler(reqpath, &globals) - if err0 == nil { - if res != nil { - payload["version"] = res.Version - } - } else { - reportable_err = err0 - } - - } else if strings.HasPrefix(reqtype, "refresh_usage-") { - res, err0 := refreshUsageHandler(reqpath, &globals) - if err0 == nil { - payload["total"] = res.Total - } else { - reportable_err = err0 - } - - } else if strings.HasPrefix(reqtype, "set_permissions-") { - reportable_err = setPermissionsHandler(reqpath, &globals) - } else if strings.HasPrefix(reqtype, "approve_probation-") { - reportable_err = approveProbationHandler(reqpath, &globals) - } else if strings.HasPrefix(reqtype, "reject_probation-") { - reportable_err = rejectProbationHandler(reqpath, &globals) - } else if strings.HasPrefix(reqtype, "create_project-") { - reportable_err = createProjectHandler(reqpath, &globals) - } else if strings.HasPrefix(reqtype, "delete_project-") { - reportable_err = deleteProjectHandler(reqpath, &globals) - } else if strings.HasPrefix(reqtype, "delete_asset-") { - reportable_err = deleteAssetHandler(reqpath, &globals) - } else if strings.HasPrefix(reqtype, "delete_version-") { - reportable_err = deleteVersionHandler(reqpath, &globals) - } else if strings.HasPrefix(reqtype, "health_check-") { - reportable_err = nil - } else { - reportable_err = fmt.Errorf("cannot determine request type for %q", reqpath) - } - - // If there's no error, or if the error is not a readRequestError, we quit - // and report the results. If the error is an RRE, we might be reading an - // incompletely written file, so we wait for a bit and try again. - if reportable_err == nil { - break - } else if _, ok := reportable_err.(*readRequestError); !ok { - break - } - time.Sleep(time.Second / 4.0) - } - - if reportable_err == nil { - payload["status"] = "SUCCESS" - } else { - log.Println(reportable_err.Error()) - payload = map[string]interface{}{ - "status": "FAILED", - "reason": reportable_err.Error(), - } - } - - err := dumpResponse(response_dir, basename, &payload) - if err != nil { - log.Println(err.Error()) - } - }(event.Name, basename) - } - } + if !strings.HasPrefix(path, "request-") { + dumpErrorResponse(w, http.StatusBadRequest, "file name should start with \"request-\"", path) + return + } + reqtype := strings.TrimPrefix(path, "request-") + + reqpath := filepath.Join(staging, path) + info, err := os.Stat(reqpath) + if err != nil { + dumpErrorResponse(w, http.StatusBadRequest, fmt.Sprintf("failed to stat; %v", err), path) + return + } + if info.IsDir() { + dumpErrorResponse(w, http.StatusBadRequest, "path is a directory", path) + return + } + + var reportable_err error + payload := map[string]interface{}{} + + if strings.HasPrefix(reqtype, "upload-") { + reportable_err = uploadHandler(reqpath, &globals) - case err, ok := <-watcher.Errors: - if !ok { - return + } else if strings.HasPrefix(reqtype, "refresh_latest-") { + res, err0 := refreshLatestHandler(reqpath, &globals) + if err0 == nil { + if res != nil { + payload["version"] = res.Version } - log.Println("watcher error;", err) + } else { + reportable_err = err0 + } + + } else if strings.HasPrefix(reqtype, "refresh_usage-") { + res, err0 := refreshUsageHandler(reqpath, &globals) + if err0 == nil { + payload["total"] = res.Total + } else { + reportable_err = err0 } + + } else if strings.HasPrefix(reqtype, "set_permissions-") { + reportable_err = setPermissionsHandler(reqpath, &globals) + } else if strings.HasPrefix(reqtype, "approve_probation-") { + reportable_err = approveProbationHandler(reqpath, &globals) + } else if strings.HasPrefix(reqtype, "reject_probation-") { + reportable_err = rejectProbationHandler(reqpath, &globals) + } else if strings.HasPrefix(reqtype, "create_project-") { + reportable_err = createProjectHandler(reqpath, &globals) + } else if strings.HasPrefix(reqtype, "delete_project-") { + reportable_err = deleteProjectHandler(reqpath, &globals) + } else if strings.HasPrefix(reqtype, "delete_asset-") { + reportable_err = deleteAssetHandler(reqpath, &globals) + } else if strings.HasPrefix(reqtype, "delete_version-") { + reportable_err = deleteVersionHandler(reqpath, &globals) + } else if strings.HasPrefix(reqtype, "health_check-") { + reportable_err = nil + } else { + dumpErrorResponse(w, http.StatusBadRequest, "invalid request type", reqpath) + return } - }() - err = watcher.Add(staging) - if err != nil { - log.Fatal(err) - } + if reportable_err == nil { + payload["status"] = "SUCCESS" + dumpJsonResponse(w, http.StatusOK, &payload, path) + } else { + status_code := http.StatusInternalServerError + var http_err *httpError + if errors.As(reportable_err, &http_err) { + status_code = http_err.Status + } + dumpErrorResponse(w, status_code, reportable_err.Error(), path) + } + }) // Adding a per-day job that purges various old files. ticker := time.NewTicker(time.Hour * 24) defer ticker.Stop() - protected := map[string]bool{} - protected[response_name] = true go func() { for { <-ticker.C - err := purgeOldFiles(staging, time.Hour * 24 * 7, protected) - if err != nil { - log.Println(err) - } - - err = purgeOldFiles(response_dir, time.Hour * 24 * 7, nil) + err := purgeOldFiles(staging, time.Hour * 24 * 7) if err != nil { log.Println(err) } - err = purgeOldFiles(log_dir, time.Hour * 24 * 7, nil) + err = purgeOldFiles(log_dir, time.Hour * 24 * 7) if err != nil { log.Println(err) } } }() - // Block main goroutine forever. - <-make(chan struct{}) + // Setting up the API. + http.ListenAndServe("0.0.0.0:" + strconv.Itoa(*port), nil) } diff --git a/permissions.go b/permissions.go index 7e4faba..5504b04 100644 --- a/permissions.go +++ b/permissions.go @@ -10,6 +10,7 @@ import ( "path/filepath" "encoding/json" "time" + "net/http" ) type uploaderEntry struct { @@ -169,17 +170,17 @@ func setPermissionsHandler(reqpath string, globals *globalConfiguration) error { { handle, err := os.ReadFile(reqpath) if err != nil { - return &readRequestError{ Cause: fmt.Errorf("failed to read %q; %w", reqpath, err) } + return fmt.Errorf("failed to read %q; %w", reqpath, err) } err = json.Unmarshal(handle, &incoming) if err != nil { - return &readRequestError{ Cause: fmt.Errorf("failed to parse JSON from %q; %w", reqpath, err) } + return newHttpError(http.StatusBadRequest, fmt.Errorf("failed to parse JSON from %q; %w", reqpath, err)) } err = isMissingOrBadName(incoming.Project) if err != nil { - return fmt.Errorf("invalid 'project' property in %q; %w", reqpath, err) + return newHttpError(http.StatusBadRequest, fmt.Errorf("invalid 'project' property in %q; %w", reqpath, err)) } } @@ -202,7 +203,7 @@ func setPermissionsHandler(reqpath string, globals *globalConfiguration) error { } if !isAuthorizedToMaintain(source_user, globals.Administrators, existing.Owners) { - return fmt.Errorf("user %q is not authorized to modify permissions for %q", source_user, project) + return newHttpError(http.StatusForbidden, fmt.Errorf("user %q is not authorized to modify permissions for %q", source_user, project)) } if incoming.Permissions.Owners != nil { @@ -211,7 +212,7 @@ func setPermissionsHandler(reqpath string, globals *globalConfiguration) error { if incoming.Permissions.Uploaders != nil { san, err := sanitizeUploaders(incoming.Permissions.Uploaders) if err != nil { - return fmt.Errorf("invalid 'permissions.uploaders' in request; %w", err) + return newHttpError(http.StatusBadRequest, fmt.Errorf("invalid 'permissions.uploaders' in request; %w", err)) } existing.Uploaders = san } diff --git a/probation.go b/probation.go index 59006fb..cd4c6e5 100644 --- a/probation.go +++ b/probation.go @@ -7,6 +7,7 @@ import ( "path/filepath" "time" "errors" + "net/http" ) func baseProbationHandler(reqpath string, globals *globalConfiguration, approve bool) error { @@ -18,27 +19,27 @@ func baseProbationHandler(reqpath string, globals *globalConfiguration, approve { handle, err := os.ReadFile(reqpath) if err != nil { - return &readRequestError{ Cause: fmt.Errorf("failed to read %q; %w", reqpath, err) } + return fmt.Errorf("failed to read %q; %w", reqpath, err) } err = json.Unmarshal(handle, &incoming) if err != nil { - return &readRequestError{ Cause: fmt.Errorf("failed to parse JSON from %q; %w", reqpath, err) } + return newHttpError(http.StatusBadRequest, fmt.Errorf("failed to parse JSON from %q; %w", reqpath, err)) } err = isMissingOrBadName(incoming.Project) if err != nil { - return &readRequestError{ Cause: fmt.Errorf("invalid 'project' property in %q; %w", reqpath, err) } + return newHttpError(http.StatusBadRequest, fmt.Errorf("invalid 'project' property in %q; %w", reqpath, err)) } err = isMissingOrBadName(incoming.Asset) if err != nil { - return &readRequestError{ Cause: fmt.Errorf("invalid 'asset' property in %q; %w", reqpath, err) } + return newHttpError(http.StatusBadRequest, fmt.Errorf("invalid 'asset' property in %q; %w", reqpath, err)) } err = isMissingOrBadName(incoming.Version) if err != nil { - return &readRequestError{ Cause: fmt.Errorf("invalid 'version' property in %q; %w", reqpath, err) } + return newHttpError(http.StatusBadRequest, fmt.Errorf("invalid 'version' property in %q; %w", reqpath, err)) } } @@ -60,7 +61,7 @@ func baseProbationHandler(reqpath string, globals *globalConfiguration, approve return fmt.Errorf("failed to read permissions for %q; %w", project_dir, err) } if !isAuthorizedToMaintain(username, globals.Administrators, existing.Owners) { - return fmt.Errorf("user %q is not authorized to modify probation status in %q", username, project_dir) + return newHttpError(http.StatusForbidden, fmt.Errorf("user %q is not authorized to modify probation status in %q", username, project_dir)) } asset_dir := filepath.Join(project_dir, *(incoming.Asset)) diff --git a/purge.go b/purge.go index 25e6531..421187e 100644 --- a/purge.go +++ b/purge.go @@ -8,7 +8,7 @@ import ( "errors" ) -func purgeOldFiles(dir string, limit time.Duration, protected map[string]bool) error { +func purgeOldFiles(dir string, limit time.Duration) error { var to_delete []string present := time.Now() messages := []string{} @@ -24,14 +24,7 @@ func purgeOldFiles(dir string, limit time.Duration, protected map[string]bool) e delta := present.Sub(info.ModTime()) if (delta > limit) { - is_protected := false - if protected != nil { - rel, _ := filepath.Rel(dir, path) - _, is_protected = protected[rel] - } - if !is_protected { - to_delete = append(to_delete, path) - } + to_delete = append(to_delete, path) } if (info.IsDir()) { diff --git a/purge_test.go b/purge_test.go index 8881691..3fd1673 100644 --- a/purge_test.go +++ b/purge_test.go @@ -49,7 +49,7 @@ func TestPurgeOldFiles (t *testing.T) { } // Deleting with a 1-hour expiry. - err = purgeOldFiles(dir, 1 * time.Hour, nil) + err = purgeOldFiles(dir, 1 * time.Hour) if (err != nil) { t.Fatal(err) } @@ -63,27 +63,8 @@ func TestPurgeOldFiles (t *testing.T) { t.Error("should not have deleted this file") } - // Deleting with an immediate expiry but also protection. - err = purgeOldFiles(dir, 0 * time.Hour, map[string]bool{ "A": true, "sub": true }) - if (err != nil) { - t.Fatal(err) - } - if _, err := os.Stat(path); errors.Is(err, os.ErrNotExist) { - t.Error("should not have deleted this file") - } - if _, err := os.Stat(subdir); errors.Is(err, os.ErrNotExist) { - t.Error("should not have deleted this directory") - } - - if _, err := os.Stat(sympath); !errors.Is(err, os.ErrNotExist) { // Symlink can be deleted, but not its target. - t.Error("should have deleted the symlink") - } - if _, err := os.Stat(target); errors.Is(err, os.ErrNotExist) { - t.Error("should not have deleted the symlink target") - } - // Deleting with an immediate expiry. - err = purgeOldFiles(dir, 0 * time.Hour, nil) + err = purgeOldFiles(dir, 0 * time.Hour) if (err != nil) { t.Fatal(err) } @@ -96,4 +77,11 @@ func TestPurgeOldFiles (t *testing.T) { if _, err := os.Stat(subdir); !errors.Is(err, os.ErrNotExist) { t.Error("should have deleted this directory") } + + if _, err := os.Stat(sympath); !errors.Is(err, os.ErrNotExist) { // Symlink can be deleted, but not its target. + t.Error("should have deleted the symlink") + } + if _, err := os.Stat(target); errors.Is(err, os.ErrNotExist) { + t.Error("should not have deleted the symlink target") + } } diff --git a/upload.go b/upload.go index 0b2f61c..d670b24 100644 --- a/upload.go +++ b/upload.go @@ -7,12 +7,13 @@ import ( "os" "encoding/json" "errors" + "net/http" ) func configureAsset(project_dir string, asset string) error { err := isBadName(asset) if err != nil { - return fmt.Errorf("invalid asset name %q; %w", asset, err) + return newHttpError(http.StatusBadRequest, fmt.Errorf("invalid asset name %q; %w", asset, err)) } asset_dir := filepath.Join(project_dir, asset) @@ -29,12 +30,12 @@ func configureAsset(project_dir string, asset string) error { func configureVersion(asset_dir string, version string) error { err := isBadName(version) if err != nil { - return fmt.Errorf("invalid version name %q; %w", version, err) + return newHttpError(http.StatusBadRequest, fmt.Errorf("invalid version name %q; %w", version, err)) } candidate_path := filepath.Join(asset_dir, version) if _, err := os.Stat(candidate_path); err == nil { - return fmt.Errorf("version %q already exists in %q", version, asset_dir) + return newHttpError(http.StatusBadRequest, fmt.Errorf("version %q already exists in %q", version, asset_dir)) } err = os.Mkdir(candidate_path, 0755) @@ -67,19 +68,19 @@ func uploadHandler(reqpath string, globals *globalConfiguration) error { { handle, err := os.ReadFile(reqpath) if err != nil { - return &readRequestError{ Cause: fmt.Errorf("failed to read %q; %w", reqpath, err) } + return fmt.Errorf("failed to read %q; %w", reqpath, err) } err = json.Unmarshal(handle, &request) if err != nil { - return &readRequestError{ Cause: fmt.Errorf("failed to parse JSON from %q; %w", reqpath, err) } + return newHttpError(http.StatusBadRequest, fmt.Errorf("failed to parse JSON from %q; %w", reqpath, err)) } if request.Source == nil { - return fmt.Errorf("expected a 'source' property in %q; %w", reqpath, err) + return newHttpError(http.StatusBadRequest, fmt.Errorf("expected a 'source' property in %q; %w", reqpath, err)) } source = *(request.Source) if source != filepath.Base(source) { - return fmt.Errorf("expected 'source' to be in the same directory as %q", reqpath) + return newHttpError(http.StatusBadRequest, fmt.Errorf("expected 'source' to be in the same directory as %q", reqpath)) } source = filepath.Join(filepath.Dir(reqpath), source) @@ -88,7 +89,7 @@ func uploadHandler(reqpath string, globals *globalConfiguration) error { return fmt.Errorf("failed to find owner of %q; %w", source, err) } if source_user != req_user { - return fmt.Errorf("requesting user must be the same as the owner of the 'source' directory (%s vs %s)", source_user, req_user) + return newHttpError(http.StatusForbidden, fmt.Errorf("requesting user must be the same as the owner of the 'source' directory (%s vs %s)", source_user, req_user)) } } @@ -96,7 +97,7 @@ func uploadHandler(reqpath string, globals *globalConfiguration) error { // Configuring the project; we apply a lock to the project to avoid concurrent changes. if request.Project == nil { - return fmt.Errorf("expected a 'project' property in %q", reqpath) + return newHttpError(http.StatusBadRequest, fmt.Errorf("expected a 'project' property in %q", reqpath)) } project := *(request.Project) @@ -113,7 +114,7 @@ func uploadHandler(reqpath string, globals *globalConfiguration) error { } ok, trusted := isAuthorizedToUpload(req_user, globals.Administrators, perms, request.Asset, request.Version) if !ok { - return fmt.Errorf("user '" + req_user + "' is not authorized to upload to '" + project + "'") + return newHttpError(http.StatusForbidden, fmt.Errorf("user '" + req_user + "' is not authorized to upload to '" + project + "'")) } if !trusted { on_probation = true @@ -121,7 +122,7 @@ func uploadHandler(reqpath string, globals *globalConfiguration) error { // Configuring the asset and version. if request.Asset == nil { - return fmt.Errorf("expected an 'asset' property in %q", reqpath) + return newHttpError(http.StatusBadRequest, fmt.Errorf("expected an 'asset' property in %q", reqpath)) } asset := *(request.Asset) @@ -132,7 +133,7 @@ func uploadHandler(reqpath string, globals *globalConfiguration) error { asset_dir := filepath.Join(project_dir, asset) if request.Version == nil { - return fmt.Errorf("expected a 'version' property in %q", reqpath) + return newHttpError(http.StatusBadRequest, fmt.Errorf("expected a 'version' property in %q", reqpath)) } version := *(request.Version) diff --git a/usage.go b/usage.go index 02d8bef..2aa29db 100644 --- a/usage.go +++ b/usage.go @@ -8,6 +8,7 @@ import ( "strings" "path/filepath" "time" + "net/http" ) type usageMetadata struct { @@ -84,7 +85,7 @@ func refreshUsageHandler(reqpath string, globals *globalConfiguration) (*usageMe } if !isAuthorizedToAdmin(source_user, globals.Administrators) { - return nil, fmt.Errorf("user %q is not authorized to refresh the latest version (%q)", source_user, reqpath) + return nil, newHttpError(http.StatusForbidden, fmt.Errorf("user %q is not authorized to refresh the latest version (%q)", source_user, reqpath)) } incoming := struct { @@ -93,17 +94,17 @@ func refreshUsageHandler(reqpath string, globals *globalConfiguration) (*usageMe { handle, err := os.ReadFile(reqpath) if err != nil { - return nil, &readRequestError{ Cause: fmt.Errorf("failed to read %q; %w", reqpath, err) } + return nil, fmt.Errorf("failed to read %q; %w", reqpath, err) } err = json.Unmarshal(handle, &incoming) if err != nil { - return nil, &readRequestError{ Cause: fmt.Errorf("failed to parse JSON from %q; %w", reqpath, err) } + return nil, newHttpError(http.StatusBadRequest, fmt.Errorf("failed to parse JSON from %q; %w", reqpath, err)) } err = isMissingOrBadName(incoming.Project) if err != nil { - return nil, fmt.Errorf("invalid 'project' property in %q; %w", reqpath, err) + return nil, newHttpError(http.StatusBadRequest, fmt.Errorf("invalid 'project' property in %q; %w", reqpath, err)) } } diff --git a/utils.go b/utils.go index 62b59e9..8edc437 100644 --- a/utils.go +++ b/utils.go @@ -26,12 +26,21 @@ func newGlobalConfiguration(registry string) globalConfiguration { } } -type readRequestError struct { - Cause error +type httpError struct { + Status int + Reason error } -func (r *readRequestError) Error() string { - return r.Cause.Error() +func (r *httpError) Error() string { + return r.Reason.Error() +} + +func (r *httpError) Unwrap() error { + return r.Reason +} + +func newHttpError(status int, reason error) *httpError { + return &httpError{ Status: status, Reason: reason } } func dumpJson(path string, content interface{}) error { @@ -105,7 +114,3 @@ func dumpLog(registry string, content interface{}) error { path := time.Now().Format(time.RFC3339) + "_" + strconv.Itoa(100000 + rand.Intn(900000)) return dumpJson(filepath.Join(registry, logDirName, path), content) } - -func dumpResponse(response_dir, reqname string, content interface{}) error { - return dumpJson(filepath.Join(response_dir, reqname), content) -} diff --git a/utils_test.go b/utils_test.go index 8c56c7c..a9a0c55 100644 --- a/utils_test.go +++ b/utils_test.go @@ -109,31 +109,3 @@ func TestIsBadName(t *testing.T) { t.Fatal("failed to stop in the presence of a backslash") } } - -func TestDumpResponse(t *testing.T) { - response_dir, err := os.MkdirTemp("", "") - if err != nil { - t.Fatalf("failed to create a temporary directory; %v", err) - } - - basename := "FOO" - payload := map[string]string { "A": "B", "C": "D" } - err = dumpResponse(response_dir, basename, &payload) - if err != nil { - t.Fatalf("failed to dump a response; %v", err) - } - - as_str, err := os.ReadFile(filepath.Join(response_dir, basename)) - if err != nil { - t.Fatalf("failed to read the response; %v", err) - } - - var roundtrip map[string]string - err = json.Unmarshal(as_str, &roundtrip) - if err != nil { - t.Fatalf("failed to parse the response; %v", err) - } - if roundtrip["A"] != payload["A"] || roundtrip["C"] != payload["C"] { - t.Fatalf("unexpected contents from roundtrip of the response") - } -}