diff --git a/.gitignore b/.gitignore index 49d3eec..387e86c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,9 +1,11 @@ -accounts*.yaml +*.csv +*.txt .vscode -output.csv +/.idea/ __debug_bin +accounts*.yaml costpuller output-* +output.csv report-* -*.csv -*.txt +*credentials*.json diff --git a/aws.go b/aws.go index e59bc2f..a674aa6 100644 --- a/aws.go +++ b/aws.go @@ -11,31 +11,33 @@ import ( "github.com/aws/aws-sdk-go/service/costexplorer" "github.com/aws/aws-sdk-go/service/organizations" "github.com/jinzhu/now" + "google.golang.org/api/sheets/v4" ) -const AWSTagCostpullerCategory = "costpuller_category" +const AwsTagCostpullerCategory = "costpuller_category" -const AWSMetadataDescription = "description" -const AWSMetadataStatus = "status" +const AwsMetadataDescription = "description" +const AwsMetadataStatus = "status" -// AWSPuller implements the AWS query client -type AWSPuller struct { +// AwsPuller implements the AWS query client +type AwsPuller struct { session *session.Session - debug bool + debug bool } -// NewAWSPuller returns a new AWS client. -func NewAWSPuller(debug bool) *AWSPuller { - awsp := new(AWSPuller) +// NewAwsPuller returns a new AWS client. +func NewAwsPuller(profile string, debug bool) *AwsPuller { + awsp := new(AwsPuller) awsp.session = session.Must(session.NewSessionWithOptions(session.Options{ - SharedConfigState: session.SharedConfigEnable, + Profile: profile, + SharedConfigState: session.SharedConfigEnable, })) awsp.debug = debug return awsp } // PullData retrieves a raw data set. -func (a *AWSPuller) PullData(accountID string, month string, costType string) (map[string]float64, error) { +func (a *AwsPuller) PullData(accountID string, month string, costType string) (map[string]float64, error) { // check month format focusMonth, err := time.Parse("2006-01", month) if err != nil { @@ -46,12 +48,9 @@ func (a *AWSPuller) PullData(accountID string, month string, costType string) (m endOfMonth := now.With(focusMonth).EndOfMonth().Add(time.Hour * 24) dayStart := beginningOfMonth.Format("2006-01-02") dayEnd := endOfMonth.Format("2006-01-02") - log.Printf("[pullawsdata] using date range %s to %s", dayStart, dayEnd) // retrieve AWS cost svc := costexplorer.New(a.session) granularity := "MONTHLY" - metricsBlendedCost := costType - log.Printf("[pullawsdata] using cost type %s", metricsBlendedCost) dimensionLinkedAccountKey := "LINKED_ACCOUNT" dimensionLinkedAccountValue := accountID groupByDimension := "DIMENSION" @@ -59,20 +58,20 @@ func (a *AWSPuller) PullData(accountID string, month string, costType string) (m costAndUsageService, err := svc.GetCostAndUsage(&costexplorer.GetCostAndUsageInput{ TimePeriod: &costexplorer.DateInterval{ Start: &dayStart, - End: &dayEnd, + End: &dayEnd, }, Granularity: &granularity, - Metrics: []*string{&metricsBlendedCost}, + Metrics: []*string{&costType}, Filter: &costexplorer.Expression{ Dimensions: &costexplorer.DimensionValues{ - Key: &dimensionLinkedAccountKey, + Key: &dimensionLinkedAccountKey, Values: []*string{&dimensionLinkedAccountValue}, }, }, GroupBy: []*costexplorer.GroupDefinition{ - &costexplorer.GroupDefinition{ + { Type: &groupByDimension, - Key: &groupByService, + Key: &groupByService, }, }, }) @@ -87,13 +86,13 @@ func (a *AWSPuller) PullData(accountID string, month string, costType string) (m costAndUsageTotal, err := svc.GetCostAndUsage(&costexplorer.GetCostAndUsageInput{ TimePeriod: &costexplorer.DateInterval{ Start: &dayStart, - End: &dayEnd, + End: &dayEnd, }, Granularity: &granularity, - Metrics: []*string{&metricsBlendedCost}, + Metrics: []*string{&costType}, Filter: &costexplorer.Expression{ Dimensions: &costexplorer.DimensionValues{ - Key: &dimensionLinkedAccountKey, + Key: &dimensionLinkedAccountKey, Values: []*string{&dimensionLinkedAccountValue}, }, }, @@ -107,13 +106,13 @@ func (a *AWSPuller) PullData(accountID string, month string, costType string) (m log.Println(*costAndUsageTotal) } // decode total value - totalAWSStr := *(*(*costAndUsageTotal.ResultsByTime[0]).Total[metricsBlendedCost]).Amount + totalAWSStr := *costAndUsageTotal.ResultsByTime[0].Total[costType].Amount totalAWS, err := strconv.ParseFloat(totalAWSStr, 64) if err != nil { log.Printf("[pullawsdata] error converting aws total value: %v", err) return nil, err } - unitAWS := *(*(*costAndUsageTotal.ResultsByTime[0]).Total[metricsBlendedCost]).Unit + unitAWS := *costAndUsageTotal.ResultsByTime[0].Total[costType].Unit if unitAWS != "USD" { log.Printf("[pullawsdata] pulled unit is not USD: %s", unitAWS) return nil, fmt.Errorf("pulled unit is not USD: %s", unitAWS) @@ -123,21 +122,35 @@ func (a *AWSPuller) PullData(accountID string, month string, costType string) (m serviceResults := make(map[string]float64) resultsByTime := costAndUsageService.ResultsByTime if len(resultsByTime) != 1 { - log.Printf("[pullawsdata] warning account %s does not have exactly one service results by time (has %d)", accountID, len(resultsByTime)) + log.Printf( + "[pullawsdata] warning account %s does not have exactly one service results by time (has %d)", + accountID, + len(resultsByTime), + ) return serviceResults, nil } serviceGroups := resultsByTime[0].Groups - for _, group := range(serviceGroups) { + for _, group := range serviceGroups { if len(group.Keys) != 1 { - log.Printf("[pullawsdata] warning account %s service group does not have exactly one key", accountID) - return serviceResults, fmt.Errorf("[pullawsdata] warning account %s service group does not have exactly one key", accountID) + err := fmt.Errorf( + "[pullawsdata] warning account %s service group does not have exactly one key", + accountID, + ) + log.Printf(err.Error()) + return serviceResults, err } key := group.Keys[0] valueStr := group.Metrics[costType].Amount unit := group.Metrics[costType].Unit if *unit != unitAWS { - log.Printf("[pullawsdata] error: inconsistent units (%s vs %s) for account %s", unitAWS, *unit, accountID) - return nil, fmt.Errorf("[pullawsdata] error: inconsistent units (%s vs %s) for account %s", unitAWS, *unit, accountID) + err := fmt.Errorf( + "[pullawsdata] error: inconsistent units (%s vs %s) for account %s", + unitAWS, + *unit, + accountID, + ) + log.Printf(err.Error()) + return nil, err } value, err := strconv.ParseFloat(*valueStr, 64) if err != nil { @@ -147,91 +160,110 @@ func (a *AWSPuller) PullData(accountID string, month string, costType string) (m serviceResults[*key] = value totalService += value } - if math.Round(totalService*100)/100 != math.Round(totalAWS*100)/100 { - log.Printf("[pullawsdata] error: account %s service total %f does not match aws total %f", accountID, totalService, totalAWS) - return nil, fmt.Errorf("[pullawsdata] error: account %s service total %f does not match aws total %f", accountID, totalService, totalAWS) + if math.Round(totalService*100)/100 != math.Round(totalAWS*100)/100 { + err := fmt.Errorf( + "[pullawsdata] error: account %s service total %f does not match aws total %f", + accountID, + totalService, + totalAWS, + ) + log.Printf(err.Error()) + return nil, err } return serviceResults, nil } // NormalizeResponse normalizes a Response object data into report categories. -func (a *AWSPuller) NormalizeResponse(group string, daterange string, accountID string, serviceResults map[string]float64) ([]string, error) { - // format is: - // group, date, clusterId, accountId, PO, clusterType, usageType, product, infra, numberUsers, dataTransfer, machines, storage, keyMgmnt, registrar, dns, other, tax, refund - - // remove: 2 4 5 6 7 9 - output := make([]string, 13) - for idx := range(output) { - output[idx] = "PENDING" - } +func (a *AwsPuller) NormalizeResponse( + group string, + dateRange string, + accountID string, + serviceResults map[string]float64, +) (*sheets.RowData, error) { + // Format is: + // [0-9] group, date, clusterId, accountId, PO, clusterType, usageType, product, infra, numberUsers, + // [10-18] dataTransfer, machines, storage, keyMgmnt, registrar, dns, other, tax, rebate + // Select entries 0, 1, 3, 8, and 10-18; omit entries 2, 4, 5, 6, 7, and 9 + output := sheets.RowData{Values: make([]*sheets.CellData, 13)} // set group - output[0] = group - // infra is always AWS - output[3] = "AWS" + output.Values[0] = newStringCell(group) // set date - we use the first service entry - output[1] = daterange - // set clusterID - output[2] = accountID - // init cost values - output[4] = "0" - output[5] = "0" - output[6] = "0" - output[7] = "0" - output[8] = "0" - output[9] = "0" - output[10] = "0" - output[11] = "0" - output[12] = "0" - // nomalize cost values + output.Values[1] = newStringCell(dateRange) + // skip clusterId; set the accountId + output.Values[2] = newStringCell(accountID) + // skip PO, clusterType, usageType, and product; infra is always AWS + output.Values[3] = newStringCell("AWS") + + // skip numberUsers; pick out and set the values for dataTransfer, storage, + // dns, and tax; sum the remaining values into categories for machines, + // keyMgmnt, and "other". var ec2Val float64 = 0 var kmVal float64 = 0 var otherVal float64 = 0 - for key, value := range(serviceResults) { + + // set default values, in case they are omitted from the data + output.Values[4] = newNumberCell(0.0) + output.Values[6] = newNumberCell(0.0) + output.Values[9] = newNumberCell(0.0) + output.Values[11] = newNumberCell(0.0) + + for key, value := range serviceResults { switch key { case "AWS Data Transfer": - output[4] = fmt.Sprintf("%f", value) + output.Values[4] = newNumberCell(value) case "Amazon Elastic Compute Cloud - Compute": ec2Val += value case "EC2 - Other": ec2Val += value case "Amazon Simple Storage Service": - output[6] = fmt.Sprintf("%f", value) + output.Values[6] = newNumberCell(value) case "AWS Key Management Service": kmVal += value case "AWS Secrets Manager": kmVal += value case "Amazon Route 53": - output[9] = fmt.Sprintf("%f", value) + output.Values[9] = newNumberCell(value) case "Tax": - output[11] = fmt.Sprintf("%f", value) + output.Values[11] = newNumberCell(value) default: otherVal += value } } - // EC2 - output[5] = fmt.Sprintf("%f", ec2Val) + // EC2 ("machines") + output.Values[5] = newNumberCell(ec2Val) // key management - output[7] = fmt.Sprintf("%f", kmVal) - // store other total - output[10] = fmt.Sprintf("%f", otherVal) - return output, nil + output.Values[7] = newNumberCell(kmVal) + // registrar (always zero??) + output.Values[8] = newNumberCell(0.0) + // "other" total + output.Values[10] = newNumberCell(otherVal) + // rebate (always zero??) + output.Values[12] = newNumberCell(0.0) + return &output, nil } // CheckResponseConsistency checks the response consistency with various checks. Returns the calculated total. -func (a *AWSPuller) CheckResponseConsistency(account AccountEntry, results map[string]float64) (float64, error) { +func (a *AwsPuller) CheckResponseConsistency(account AccountEntry, results map[string]float64) (float64, error) { var total float64 = 0 - for _, value := range(results) { + for _, value := range results { // add up value total += value } - // check account meta deviation if standardvalue is given + // check account meta deviation if standard value is given if account.Standardvalue > 0 { diff := account.Standardvalue - total diffAbs := math.Abs(diff) diffPercent := (diffAbs / account.Standardvalue) * 100 if diffPercent > float64(account.Deviationpercent) { - return total, fmt.Errorf("deviation check failed: deviation is %.2f (%.2f%%), max deviation allowed is %d%% (value was %.2f, standard value %.2f)", diffAbs, diffPercent, account.Deviationpercent, total, account.Standardvalue) - } + return total, fmt.Errorf( + "deviation check failed: deviation is %.2f (%.2f%%), max deviation allowed is %d%% (value was %.2f, standard value %.2f)", + diffAbs, + diffPercent, + account.Deviationpercent, + total, + account.Standardvalue, + ) + } } if a.debug { log.Println("[CheckResponseConsistency] service struct:") @@ -241,19 +273,19 @@ func (a *AWSPuller) CheckResponseConsistency(account AccountEntry, results map[s return total, nil } -// GetAWSAccountMetadata returns a map with accountIDs as keys and metadata key-value pairs map as value. -func (a *AWSPuller) GetAWSAccountMetadata() (map[string]map[string]string, error) { +// GetAwsAccountMetadata returns a map with accountIDs as keys and metadata key-value pairs map as value. +func (a *AwsPuller) GetAwsAccountMetadata() (map[string]map[string]string, error) { // get account list and basic metadata accounts, err := a.getAllAWSAccountData() if err != nil { return nil, err } // augment tags - log.Println("[GetAWSAccountMetadata] starting tags pull for accounts") + log.Println("[GetAwsAccountMetadata] starting tags pull for accounts") idx := 0 - for accountID, _ := range accounts { + for accountID := range accounts { idx++ - log.Printf("[GetAWSAccountMetadata] pulling tags for account %s (%d of %d)", accountID, idx, len(accounts)) + log.Printf("[GetAwsAccountMetadata] pulling tags for account %s (%d of %d)", accountID, idx, len(accounts)) tags, err := a.getTagsForAWSAccount(accountID) if err != nil { @@ -266,7 +298,7 @@ func (a *AWSPuller) GetAWSAccountMetadata() (map[string]map[string]string, error return accounts, nil } -func (a *AWSPuller) getTagsForAWSAccount(accountID string) (map[string]string, error) { +func (a *AwsPuller) getTagsForAWSAccount(accountID string) (map[string]string, error) { result := map[string]string{} svo := organizations.New(a.session) output, err := svo.ListTagsForResource(&organizations.ListTagsForResourceInput{ @@ -296,7 +328,11 @@ func (a *AWSPuller) getTagsForAWSAccount(accountID string) (map[string]string, e return result, nil } -func (a *AWSPuller) pullAccountData(svo *organizations.Organizations, result *map[string]map[string]string, nextToken *string) (*string, error) { +func (a *AwsPuller) pullAccountData( + svo *organizations.Organizations, + result *map[string]map[string]string, + nextToken *string, +) (*string, error) { limit := int64(10) output, err := svo.ListAccounts(&organizations.ListAccountsInput{ MaxResults: &limit, @@ -308,14 +344,14 @@ func (a *AWSPuller) pullAccountData(svo *organizations.Organizations, result *ma } for _, e := range output.Accounts { (*result)[*e.Id] = map[string]string{ - AWSMetadataDescription: *e.Name, - AWSMetadataStatus: *e.Status, + AwsMetadataDescription: *e.Name, + AwsMetadataStatus: *e.Status, } - } + } return output.NextToken, nil } -func (a *AWSPuller) getAllAWSAccountData() (map[string]map[string]string, error) { +func (a *AwsPuller) getAllAWSAccountData() (map[string]map[string]string, error) { result := map[string]map[string]string{} svo := organizations.New(a.session) log.Println("[pullawsdata] pulling all accounts metadata") @@ -335,22 +371,19 @@ func (a *AWSPuller) getAllAWSAccountData() (map[string]map[string]string, error) return result, nil } -func (a *AWSPuller) WriteAWSTags(accounts map[string][]AccountEntry) (error) { +func (a *AwsPuller) WriteAwsTags(accounts map[string][]AccountEntry) error { svo := organizations.New(a.session) - catgoryTag := AWSTagCostpullerCategory + categoryTag := AwsTagCostpullerCategory for category, accountEntries := range accounts { for _, accountEntry := range accountEntries { - fmt.Printf("setting tag %s == %s for account %s...", catgoryTag, category, accountEntry.AccountID) + fmt.Printf("setting tag %s == %s for account %s...", categoryTag, category, accountEntry.AccountID) if !a.debug { _, err := svo.TagResource(&organizations.TagResourceInput{ ResourceId: &accountEntry.AccountID, - Tags: []*organizations.Tag{ - &organizations.Tag{ - Key: &catgoryTag, - Value: &category, - }, + Tags: []*organizations.Tag{ + {Key: &categoryTag, Value: &category}, }, - }) + }) if err != nil { return err } @@ -362,3 +395,11 @@ func (a *AWSPuller) WriteAWSTags(accounts map[string][]AccountEntry) (error) { } return nil } + +func newStringCell(val string) *sheets.CellData { + return &sheets.CellData{UserEnteredValue: &sheets.ExtendedValue{StringValue: &val}} +} + +func newNumberCell(val float64) *sheets.CellData { + return &sheets.CellData{UserEnteredValue: &sheets.ExtendedValue{NumberValue: &val}} +} diff --git a/costmanagement.go b/costmanagement.go index 78ec1b6..9d557b5 100644 --- a/costmanagement.go +++ b/costmanagement.go @@ -4,7 +4,7 @@ import ( "encoding/json" "errors" "fmt" - "io/ioutil" + "io" "log" "math" "net/http" @@ -63,26 +63,30 @@ type ValueSection struct { Cost CostSection `json:"cost"` } -// CMPuller implements the Cost Management query client. -type CMPuller struct { +// CmPuller implements the Cost Management query client. +type CmPuller struct { debug bool httpClient *http.Client cookieMap map[string]string } -// NewCMPuller returns a new Cost Management client. -func NewCMPuller(debug bool, client *http.Client, cookieMap map[string]string) *CMPuller { - cmp := new(CMPuller) +// NewCmPuller returns a new Cost Management client. +func NewCmPuller(cookieMap map[string]string, debug bool) *CmPuller { + cmp := new(CmPuller) cmp.debug = debug - cmp.httpClient = client + cmp.httpClient = &http.Client{} cmp.cookieMap = cookieMap return cmp } // PullData retrieves a raw data set. -func (c *CMPuller) PullData(accountID string) ([]byte, error) { +func (c *CmPuller) PullData(accountID string) ([]byte, error) { // create request - req, err := http.NewRequest("GET", "https://cloud.redhat.com/api/cost-management/v1/reports/aws/costs/", nil) + req, err := http.NewRequest( + "GET", + "https://cloud.redhat.com/api/cost-management/v1/reports/aws/costs/", + nil, + ) if err != nil { log.Printf("[pulldata] error creating request: %v ", err) return nil, err @@ -117,12 +121,17 @@ func (c *CMPuller) PullData(accountID string) ([]byte, error) { // check response if resp.StatusCode != 200 { log.Println("[pulldata] error pulling data from server") - bodyBytes, _ := ioutil.ReadAll(resp.Body) + bodyBytes, _ := io.ReadAll(resp.Body) bodyStr := string(bodyBytes) - return nil, fmt.Errorf("error fetching data from service, returned status %d, url was %s\nBody: %s", resp.StatusCode, req.URL.String(), bodyStr) + return nil, fmt.Errorf( + "error fetching data from service, returned status %d, url was %s\nBody: %s", + resp.StatusCode, + req.URL.String(), + bodyStr, + ) } // read body - bodyBytes, err := ioutil.ReadAll(resp.Body) + bodyBytes, err := io.ReadAll(resp.Body) if err != nil { log.Printf("[pulldata] error reading body data: %v ", err) return nil, err @@ -132,7 +141,7 @@ func (c *CMPuller) PullData(accountID string) ([]byte, error) { } // ParseResponse parses a raw response into a Response object. -func (c *CMPuller) ParseResponse(response []byte) (*Response, error) { +func (c *CmPuller) ParseResponse(response []byte) (*Response, error) { responseData := new(Response) err := json.Unmarshal(response, responseData) if err != nil { @@ -143,9 +152,10 @@ func (c *CMPuller) ParseResponse(response []byte) (*Response, error) { } // NormalizeResponse normalizes a Response object data into report categories. -func (c *CMPuller) NormalizeResponse(response *Response) ([]string, error) { +func (c *CmPuller) NormalizeResponse(response *Response) ([]string, error) { // format is: - // date, clusterId, accountId, PO, clusterType, usageType, product, infra, numberUsers, dataTransfer, machines, storage, keyMgmnt, registrar, dns, other, tax, refund + // date, clusterId, accountId, PO, clusterType, usageType, product, infra, numberUsers, + // dataTransfer, machines, storage, keyMgmnt, registrar, dns, other, tax, refund // init fields with pending flag output := make([]string, 18) for idx := range output { @@ -167,7 +177,7 @@ func (c *CMPuller) NormalizeResponse(response *Response) ([]string, error) { output[15] = "0" output[16] = "0" output[17] = "0" - // nomalize cost values + // normalize cost values var otherVal float64 = 0 for _, service := range response.Data[0].Services { switch service.Service { @@ -192,7 +202,7 @@ func (c *CMPuller) NormalizeResponse(response *Response) ([]string, error) { } // CheckResponseConsistency checks the response consistency with various checks. Returns the calculated total. -func (c *CMPuller) CheckResponseConsistency(account AccountEntry, response *Response) (float64, error) { +func (c *CmPuller) CheckResponseConsistency(account AccountEntry, response *Response) (float64, error) { // TODO check base value consistence by comparing to a rough value given in the config // check that there is exactly one entry in toplevel data if len(response.Data) != 1 { @@ -202,36 +212,61 @@ func (c *CMPuller) CheckResponseConsistency(account AccountEntry, response *Resp if len(response.Data[0].Services) == 0 { return 0, errors.New("services array is empty") } - var foundDate string = response.Data[0].Date - var foundUnit string = response.Meta.Total.Cost.TotalCost.Unit + foundDate := response.Data[0].Date + foundUnit := response.Meta.Total.Cost.TotalCost.Unit var total float64 = 0 for _, service := range response.Data[0].Services { // check that there is exactly one value section in services if len(service.Values) != 1 { - return 0, fmt.Errorf("service %s has more than exactly one values section (length is %d)", service.Service, len(service.Values)) + return 0, fmt.Errorf( + "service %s has more than exactly one values section (length is %d)", + service.Service, + len(service.Values), + ) } // check date consistency if foundDate != service.Values[0].Date { - return 0, fmt.Errorf("service %s date stamp differs (%s vs %s)", service.Service, service.Values[0].Date, foundDate) + return 0, fmt.Errorf( + "service %s date stamp differs (%s vs %s)", + service.Service, + service.Values[0].Date, + foundDate, + ) } // check unit consistency if foundUnit != service.Values[0].Cost.TotalCost.Unit { - return 0, fmt.Errorf("service %s unit differs (%s vs %s)", service.Service, service.Values[0].Cost.TotalCost.Unit, foundUnit) + return 0, fmt.Errorf( + "service %s unit differs (%s vs %s)", + service.Service, + service.Values[0].Cost.TotalCost.Unit, + foundUnit, + ) } // add up value total += service.Values[0].Cost.TotalCost.Value } // check totals of all services is same as total in meta if math.Round(total*100)/100 != math.Round(response.Meta.Total.Cost.TotalCost.Value*100)/100 { - return 0, fmt.Errorf("total cost differs from meta and total of services (%f vs %f)", response.Meta.Total.Cost.TotalCost.Value, total) + return 0, fmt.Errorf( + "total cost differs from meta and total of services (%f vs %f)", + response.Meta.Total.Cost.TotalCost.Value, + total, + ) } - // check account meta deviation if standardvalue is given + // check account meta deviation if standard value is given if account.Standardvalue > 0 { diff := account.Standardvalue - total diffAbs := math.Abs(diff) diffPercent := (diffAbs / account.Standardvalue) * 100 if diffPercent > float64(account.Deviationpercent) { - return total, fmt.Errorf("deviation check failed: deviation is %.2f (%.2f%%), max deviation allowed is %d%% (value was %.2f, standard value %.2f)", diffAbs, diffPercent, account.Deviationpercent, total, account.Standardvalue) + return total, fmt.Errorf( + "deviation check failed: deviation is %.2f (%.2f%%), max deviation allowed is %d%% (value was %.2f, standard value %.2f)", + diffAbs, + diffPercent, + account.Deviationpercent, + total, + account.Standardvalue, + ) } } return total, nil diff --git a/costpuller.go b/costpuller.go index 6c81ffc..08497fb 100644 --- a/costpuller.go +++ b/costpuller.go @@ -1,3 +1,88 @@ +// Theory of Operation +// +// This tool gathers billing data from various accounts on various cloud +// providers. Ultimately, it will support AWS, Azure, Google Cloud Platform, +// and IBM Cloud; currently, it supports only AWS. The data gathered is either +// saved to a local file as CSV or it is loaded into a Google Sheet. +// +// The configuration for this tool is provided by a YAML file. The file +// provides the list of cloud providers and the account IDs for each one, +// grouped by organization. It also provides a section for configuring and +// customizing the operation of this tool. +// +// Providing Credentials +// +// - AWS access is controlled in the conventional ways: either via +// environment variables AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY or via +// ~/.aws/ config files created by the `awscli configure` command. If using +// the file-based credentials, you may select a specific profile. +// - Google Sheets access is provided via OAuth 2.0. This tool acts as an +// OAuth client. The client configuration is provided in the conventional +// location (${HOME}/.config/gcloud/application_default_credentials.json) +// and can be downloaded from https://console.developers.google.com, under +// "Credentials"). The access token and refresh token are cached in a local +// file. If the token file doesn't exist, this tool prompts the user to +// open a page in their browser (this should be done on the same machine +// which is running this tool). The browser will allow the user to interact +// with Google's authentication servers, and then it will be redirected to a +// listener provided by this tool, which allows the tool to obtain the +// OAuth access code. The tool then exchanges that for the tokens, which it +// writes to the cache file. +// +// The Output +// +// This tool collects the billing data from the cloud provider for each +// account in the YAML file. The data is post-processed with certain values +// being coalesced into category values. The result is a single row with +// canonical columns. The data can be output to a CSV file, or it can be +// loaded into a Google Spreadsheet. +// +// The Google Sheets Spreadsheet Configuration & Magic +// +// The Google spreadsheet is selected by its ID which is configured in the +// "gsheet" subsection of the "configuration" section of the YAML file with +// the key, "spreadsheetId". The value comes from the URL used to view the +// spreadsheet. +// +// The raw data is loaded into a new "tab" or "sheet" in the spreadsheet. +// The sheet is named by expanding a name-template configured in the YAML +// file with the key "sheetNameTemplate". Digits in the value are replaced +// with elements of the reference time timestamp, as described in +// https://pkg.go.dev/time#Layout: for instance, in "Raw Data 01/2006", the +// "01" would be replaced by the two-digit numerical month and "2006" would +// be replaced by the four-digit year. The reference time can be specified +// with the `-month` command line option, as "-month 2024-08", but it +// defaults to the month previous to the current one. +// +// The target sheet for the raw data is created by copying an existing +// template sheet in the spreadsheet whose name is configured with the key, +// "templateSheetName". The first row of this sheet is reserved for column +// headers, and so the data is written starting at the second row. The last +// column of the sheet is reserved for row totals, and so the data rows must +// fit in the preceding columns. Also, in order for the tool to correctly +// determine the "totals" column, the template sheet must not include any +// columns after the "totals" column (any extra columns should be explicitly +// deleted). +// +// Finally, the tool expects that the spreadsheet contains a "main sheet" +// which references the raw data sheets. This sheet must be specified in +// the using YAML file using the key, "mainSheetName". Unfortunately, +// Google Sheets seems to have a mal-feature which results in situations +// where references between sheets are not updated reliably. For instance, +// creating a new sheet or, in many cases, even just updating it, will not +// refresh a reference to it in another sheet. The accepted workaround for +// this is to copy and paste the cell references over themselves. To effect +// this, the tool expects that there is a cell in the main sheet which +// contains the name of the raw data sheet and which is used for indirect +// lookups in the raw data sheet, moreover that the formulas containing the +// indirect references are found in the column immediately below this cell +// and that there is one entry for each row of data. The tool will locate +// the cell which contains the sheet reference, copy the appropriate number +// of cells below it, and paste those values over themselves. The paste +// operation is non-destructive, so it is not a problem if it encompasses +// unrelated cells, but it must include all cells with references to the +// new sheet. + package main import ( @@ -6,203 +91,354 @@ import ( "errors" "flag" "fmt" - "io/ioutil" "log" "math" "net/http" "os" - "os/user" + "path/filepath" + "runtime" "sort" "strings" "time" - "github.com/zellyn/kooky" + "github.com/browserutils/kooky" + "github.com/browserutils/kooky/browser/chrome" + "google.golang.org/api/sheets/v4" "gopkg.in/yaml.v2" ) +type CommandLineOptions struct { + modePtr *string + debugPtr *bool + awsWriteTagsPtr *bool + awsCheckTagsPtr *bool + accountsFilePtr *string + taggedAccountsPtr *bool + monthPtr *string + costTypePtr *string + cookiePtr *string + readcookiePtr *bool + cookieDbPtr *string + csvfilePtr *string + reportFilePtr *string + outputTypePtr *string +} + +type AccountsFile struct { + Configuration map[string]map[string]string `yaml:"configuration"` + Providers map[string]map[string][]AccountEntry `yaml:"cloud_providers"` +} + // AccountEntry describes an account with metadata. type AccountEntry struct { - AccountID string `yaml:"accountid"` - Standardvalue float64 `yaml:"standardvalue"` - Deviationpercent int `yaml:"deviationpercent"` - Category string `yaml:"category"` - Description string `yaml:"description"` + AccountID string `yaml:"accountid"` + Standardvalue float64 `yaml:"standardvalue"` + Deviationpercent int `yaml:"deviationpercent"` + Category string `yaml:"category"` + Description string `yaml:"description"` } func main() { - var err error log.Println("[main] costpuller starting..") - // bootstrap - usr, _ := user.Current() - nowStr := time.Now().Format("20060102150405") - // configure flags - modePtr := flag.String("mode", "aws", "run mode, needs to be one of aws, cm or crosscheck") - debugPtr := flag.Bool("debug", false, "outputs debug info") - awsWriteTagsPtr := flag.Bool("awswritetags", false, "write tags to AWS accounts (USE WITH CARE!)") - awsCheckTagsPtr := flag.Bool("checktags", false, "checks all AWS accounts available for correct tag setting.") - accountsFilePtr := flag.String("accounts", "accounts.yaml", "file to read accounts list from") - taggedAccountsPtr := flag.Bool("taggedaccounts", false, "use the AWS tags as account list source") - monthPtr := flag.String("month", "", "context month in format yyyy-mm, only for aws or crosscheck modes") - costTypePtr := flag.String("costtype", "UnblendedCost", "cost type to pull, only for aws or crosscheck modes, one of AmortizedCost, BlendedCost, NetAmortizedCost, NetUnblendedCost, NormalizedUsageAmount, UnblendedCost, and UsageQuantity") - cookiePtr := flag.String("cookie", "", "access cookie for cost management system in curl serialized format, only for cm or crosscheck modes") - readcookiePtr := flag.Bool("readcookie", true, "reads the cookie from the Chrome cookies database, only for cm or crosscheck modes") - cookieDbPtr := flag.String("cookiedb", fmt.Sprintf("%s/.config/google-chrome/Default/Cookies", usr.HomeDir), "path to Chrome cookies database file, only for cm or crosscheck modes") - csvfilePtr := flag.String("csv", fmt.Sprintf("output-%s.csv", nowStr), "output file for csv data") - reportfilePtr := flag.String("report", fmt.Sprintf("report-%s.txt", nowStr), "output file for data consistency report") - flag.Parse() - // create aws puller instance - awsPuller := NewAWSPuller(*debugPtr) - if *awsWriteTagsPtr { - // we pull accounts from file - accounts, err := getAccountSetsFromFile(*accountsFilePtr) - if err != nil { - log.Fatalf("[main] error getting accounts list: %v", err) - } - err = awsPuller.WriteAWSTags(accounts) - if err != nil { - log.Fatalf("[main] error writing account tag: %v", err) - } - os.Exit(0) + nowTime := time.Now() + lastMonth := time.Date(nowTime.Year(), nowTime.Month()-1, 1, 0, 0, 0, 0, nowTime.Location()) + nowStr := nowTime.Format("20060102150405") + defaultMonth := lastMonth.Format("2006-01") + defaultCsvFile := fmt.Sprintf("output-%s.csv", defaultMonth) + defaultReportFile := fmt.Sprintf("report-%s.txt", nowStr) + options := CommandLineOptions{ + accountsFilePtr: flag.String("accounts", "accounts.yaml", "file to read accounts list from"), + awsCheckTagsPtr: flag.Bool("checktags", false, "checks all AWS accounts available for correct tag setting."), + awsWriteTagsPtr: flag.Bool("awswritetags", false, "write tags to AWS accounts (USE WITH CARE!)"), + cookieDbPtr: flag.String("cookiedb", getDefaultCookieStore(), `path to Chrome cookies database file, only for "cm" or "crosscheck" modes`), + cookiePtr: flag.String("cookie", "", `access cookie for cost management system in curl serialized format, only for "cm" or "crosscheck" modes`), + costTypePtr: flag.String("costtype", "UnblendedCost", `cost type to pull, only for "aws" or "crosscheck" modes, one of "AmortizedCost", "BlendedCost", "NetAmortizedCost", "NetUnblendedCost", "NormalizedUsageAmount", "UnblendedCost", or "UsageQuantity"`), + csvfilePtr: flag.String("csv", defaultCsvFile, "output file for csv data"), + debugPtr: flag.Bool("debug", false, "outputs debug info"), + modePtr: flag.String("mode", "aws", `run mode, needs to be one of "aws", "cm" or "crosscheck"`), + monthPtr: flag.String("month", defaultMonth, `context month in format yyyy-mm, only for "aws" or "crosscheck" modes`), + outputTypePtr: flag.String("output", "gsheet", `output destination, needs to be one of "csv" or "gsheet"`), + readcookiePtr: flag.Bool("readcookie", true, `reads the cookie from the Chrome cookies database, only for "cm" or "crosscheck" modes`), + reportFilePtr: flag.String("report", defaultReportFile, "output file for data consistency report"), + taggedAccountsPtr: flag.Bool("taggedaccounts", false, "use the AWS tags as account list source"), } - if *awsCheckTagsPtr { - log.Println("[main] checking tags on AWS") - _, err := getAccountSetsFromAWS(awsPuller) - if err != nil { - log.Fatalf("[main] error getting accounts list: %v", err) - } - os.Exit(0) + flag.Parse() + + accountsFile, err := loadAccountsFile(*options.accountsFilePtr) + if err != nil { + log.Fatalf("[main] error loading accounts file: %v", err) } - // open output files - log.Printf("[main] using csv output file %s\n", *csvfilePtr) - log.Printf("[main] using report output file %s\n", *reportfilePtr) - // create data holder - csvData := make([][]string, 0) - // get account lists - var accounts map[string][]AccountEntry - if *taggedAccountsPtr { - accounts, err = getAccountSetsFromAWS(awsPuller) - } else { - // we pull accounts from file - accounts, err = getAccountSetsFromFile(*accountsFilePtr) + if len(accountsFile.Configuration) == 0 { + log.Fatalf("[main] error in accounts file: empty or missing \"configuration\" section") } - if err != nil { - log.Fatalf("[main] error getting accounts list: %v", err) + if len(accountsFile.Providers) == 0 { + log.Fatalf("[main] error in accounts file: empty or missing \"cloud_providers\" section") } - sortedAccountKeys := sortedKeys(accounts) - if err != nil { - log.Fatalf("[main] error unmarshalling accounts file: %v", err) + + awsConfig := getMapKeyValue(accountsFile.Configuration, "aws", "configuration") + awsProfile := awsConfig["profile"] + if awsProfile == "" { + awsProfile = "default" + log.Printf( + "[main] no \"profile\" key found in the \"aws\" section of the configuration file; "+ + "using AWS credentials profile %q", + awsProfile, + ) } - // open csv output file - outfile, err := os.Create(*csvfilePtr) - if err != nil { - log.Fatalf("[main] error creating output file: %v", err) + awsPuller := NewAwsPuller(awsProfile, *options.debugPtr) + + if *options.awsWriteTagsPtr { + writeAwsTags(awsPuller, options) + os.Exit(0) } - defer outfile.Close() - // open report file - reportfile, err := os.Create(*reportfilePtr) - if err != nil { - log.Fatalf("[main] error creating report file: %v", err) + + if *options.awsCheckTagsPtr { + checkAwsTags(awsPuller) + os.Exit(0) } - defer reportfile.Close() - // check for run mode - switch *modePtr { + + reportFile := getReportFile(options) + defer closeFile(reportFile) + + awsAccounts, sortedAccountKeys := awsPuller.getAwsAccounts(accountsFile, options) + + switch *options.modePtr { case "aws": - log.Println("[main] note: using credentials and account from env AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY for aws pull") - if *monthPtr == "" || *costTypePtr == "" { - log.Fatal("[main] aws mode requested, but no month and/or costtype given (use --month=yyyy-mm, --costtype=type)") + var client *http.Client + var outfile *os.File + + refTime, err := time.Parse("2006-01", *options.monthPtr) + if err != nil { + log.Fatalf("[main] error parsing month value, %q: %v", *options.monthPtr, err) } - for _, accountKey := range(sortedAccountKeys) { - group := accountKey - accountList := accounts[accountKey] - //csvData = appendCSVHeader(csvData, group) - for _, account := range(accountList) { - log.Printf("[main] pulling data for account %s (group %s)\n", account.AccountID, group) - csvData, _, err = pullAWS(*awsPuller, reportfile, group, account, csvData, *monthPtr, *costTypePtr) - if err != nil { - log.Fatalf("[main] error pulling data: %v", err) - } + + if *options.outputTypePtr == "csv" { + outfile = getCsvFile(options) + defer closeFile(outfile) + } else if *options.outputTypePtr == "gsheet" { + oauthConfig := getMapKeyValue(accountsFile.Configuration, "oauth", "configuration") + client = getGoogleOAuthHttpClient(oauthConfig) + } else { + log.Fatalf("[main] Unexpected value for output type, %q", *options.outputTypePtr) + } + + sheetData := awsPuller.pullAwsByAccount(awsAccounts, sortedAccountKeys, options, reportFile) + + if *options.outputTypePtr == "csv" { + err = writeCsvFromSheet(outfile, sheetData) + if err != nil { + log.Fatalf("[main] error writing to output file: %v", err) } + } else if *options.outputTypePtr == "gsheet" { + gsheetConfig := getMapKeyValue(accountsFile.Configuration, "gsheet", "base") + postToGSheet(sheetData, client, gsheetConfig, refTime) } + case "cm": - cookie, err := retrieveCookie(*cookiePtr, *readcookiePtr, *cookieDbPtr) + var csvData [][]string + cookie, err := retrieveCookie(*options.cookiePtr, *options.readcookiePtr, *options.cookieDbPtr) if err != nil { log.Fatalf("[main] error retrieving cookie: %v", err) } - httpClient := &http.Client{} - cmPuller := NewCMPuller(*debugPtr, httpClient, cookie) - for _, accountKey := range(sortedAccountKeys) { + outfile := getCsvFile(options) + defer closeFile(outfile) + cmPuller := NewCmPuller(cookie, *options.debugPtr) + for _, accountKey := range sortedAccountKeys { group := accountKey - accountList := accounts[accountKey] - //csvData = appendCSVHeader(csvData, group) - for _, account := range(accountList) { - log.Printf("[main] pulling data for account %s (group %s)\n", account.AccountID, group) - csvData, _, err = pullCostManagement(*cmPuller, reportfile, account, csvData, *monthPtr) + accountList := awsAccounts[accountKey] + for _, account := range accountList { + log.Printf("[main] pulling data for account %s (group %s)\n", account.AccountID, group) + csvData, _, err = pullCostManagement(*cmPuller, reportFile, account, csvData) if err != nil { log.Fatalf("[main] error pulling data: %v", err) } } } + err = writeCsv(outfile, csvData) + if err != nil { + log.Fatalf("[main] error writing to output file: %v", err) + } case "crosscheck": - log.Println("[main] note: using credentials and account from env AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY for aws pull") - if *monthPtr == "" || *costTypePtr == "" { - log.Fatal("[main] aws mode requested, but no month and/or costtype given (use --month=yyyy-mm, --costtype=type)") + var csvData [][]string + if *options.monthPtr == "" || *options.costTypePtr == "" { + log.Fatal("[main] aws mode requested, but no month and/or cost type given (use --month=yyyy-mm, --costtype=type)") } - cookie, err := retrieveCookie(*cookiePtr, *readcookiePtr, *cookieDbPtr) + outfile := getCsvFile(options) + defer closeFile(outfile) + cookie, err := retrieveCookie(*options.cookiePtr, *options.readcookiePtr, *options.cookieDbPtr) if err != nil { log.Fatalf("[main] error retrieving cookie: %v", err) } - httpClient := &http.Client{} - cmPuller := NewCMPuller(*debugPtr, httpClient, cookie) - for _, accountKey := range(sortedAccountKeys) { + cmPuller := NewCmPuller(cookie, *options.debugPtr) + for _, accountKey := range sortedAccountKeys { group := accountKey - accountList := accounts[accountKey] - //csvData = appendCSVHeader(csvData, group) - for _, account := range(accountList) { + accountList := awsAccounts[accountKey] + for _, account := range accountList { log.Printf("[main] pulling data for account %s (group %s)\n", account.AccountID, group) - var totalAWS float64 - _, totalAWS, err = pullAWS(*awsPuller, reportfile, group, account, nil, *monthPtr, *costTypePtr) + var totalAws float64 + _, totalAws, err = awsPuller.pullAwsAccount( + account, + group, + *options.monthPtr, + *options.costTypePtr, + reportFile, + ) if err != nil { log.Fatalf("[main] error pulling data: %v", err) } var totalCM float64 - csvData, totalCM, err = pullCostManagement(*cmPuller, reportfile, account, csvData, *monthPtr) + csvData, totalCM, err = pullCostManagement(*cmPuller, reportFile, account, csvData) if err != nil { log.Fatalf("[main] error pulling data: %v", err) } // check if totals from AWS and CM are consistent - if math.Round(totalAWS*100)/100 != math.Round(totalCM*100)/100 { - log.Printf("[main] error checking consistency of totals from AWS and CM for account %s: aws = %f; cm = %f", account.AccountID, totalAWS, totalCM) - writeReport(reportfile, fmt.Sprintf("%s: error checking consistency of totals from AWS and CM: aws = %f; cm = %f", account.AccountID, totalAWS, totalCM)) + if math.Round(totalAws*100)/100 != math.Round(totalCM*100)/100 { + log.Printf( + "[main] error checking consistency of totals from AWS and CM for account %s: aws = %f; cm = %f", + account.AccountID, + totalAws, + totalCM, + ) + writeReport(reportFile, fmt.Sprintf( + "%s: error checking consistency of totals from AWS and CM: aws = %f; cm = %f", + account.AccountID, + totalAws, + totalCM, + )) } } } + err = writeCsv(outfile, csvData) + if err != nil { + log.Fatalf("[main] error writing to output file: %v", err) + } + } + + log.Println("[main] operation done") +} + +// getDefaultCookieStore encapsulates the platform-specific location of the +// default browser cookie database file. +// +// TODO: kooky.FindAllCookieStores() can handle this for us. +func getDefaultCookieStore() string { + defaultCookieDb, _ := os.UserConfigDir() + if runtime.GOOS == "linux" { + defaultCookieDb = filepath.Join(defaultCookieDb, "google-chrome") + } else if runtime.GOOS == "darwin" { + defaultCookieDb = filepath.Join(defaultCookieDb, "Google/Chrome") + } else { + log.Printf("[main] unexpected platform: %q\n", runtime.GOOS) + } + defaultCookieDb = filepath.Join(defaultCookieDb, "Default/Cookies") + return defaultCookieDb +} + +func (a *AwsPuller) getAwsAccounts( + accountsFile AccountsFile, + options CommandLineOptions, +) (accounts map[string][]AccountEntry, keys []string) { + //var accounts map[string][]AccountEntry + if *options.taggedAccountsPtr { + a, err := getAccountSetsFromAws(a) + if err != nil { + log.Fatalf("[getAwsAccounts] error getting accounts list: %v", err) + } + accounts = a + } else { + accounts = getMapKeyValue(accountsFile.Providers, "aws", "cloud_providers") + } + if len(accounts) == 0 { + fmt.Println("[getAwsAccounts] Warning: No AWS accounts found!") + } + return accounts, sortedKeys(accounts) +} + +func (a *AwsPuller) pullAwsByAccount( + accounts map[string][]AccountEntry, + sortedAccountKeys []string, + options CommandLineOptions, + reportFile *os.File, +) (sheetData []*sheets.RowData) { + if *options.monthPtr == "" || *options.costTypePtr == "" { + log.Fatal("[pullAwsByAccount] aws mode requested, but no month and/or cost type given (use --month=yyyy-mm, --costtype=type)") + } + for _, group := range sortedAccountKeys { + accountList := accounts[group] + if len(accountList) == 0 { + log.Printf("[pullAwsByAccount] Warning: no accounts found in group %q!", group) + } + for _, account := range accountList { + log.Printf("[pullAwsByAccount] pulling data for account %s (group %s)\n", account.AccountID, group) + rowData, _, err := a.pullAwsAccount( + account, + group, + *options.monthPtr, + *options.costTypePtr, + reportFile, + ) + if err != nil { + log.Fatalf("[pullAwsByAccount] error pulling data: %v", err) + } + sheetData = append(sheetData, rowData) + } + } + return +} + +func writeAwsTags(awsPuller *AwsPuller, options CommandLineOptions) { + accountsFile, err := loadAccountsFile(*options.accountsFilePtr) + if err != nil { + log.Fatalf("[writeAwsTags] error getting accounts list: %v", err) } - // write data to csv - err = writeCSV(outfile, csvData) + accounts := getMapKeyValue(accountsFile.Providers, "aws", "cloud_providers") + err = awsPuller.WriteAwsTags(accounts) if err != nil { - log.Fatalf("[main] error writing to output file: %v", err) + log.Fatalf("[writeAwsTags] error writing account tag: %v", err) + } +} + +func checkAwsTags(awsPuller *AwsPuller) { + log.Println("[checkAwsTags] checking tags on AWS") + _, err := getAccountSetsFromAws(awsPuller) + if err != nil { + log.Fatalf("[checkAwsTags] error getting accounts list: %v", err) } - // done - log.Println("[main] operation done") } -func sortedKeys(m map[string][]AccountEntry) ([]string) { - keys := make([]string, len(m)) - i := 0 +func getCsvFile(options CommandLineOptions) *os.File { + outfile, err := os.Create(*options.csvfilePtr) + if err != nil { + log.Fatalf("[getCsvFile] error creating output file: %v", err) + } + log.Printf("[getCsvFile] using csv output file %s\n", *options.csvfilePtr) + return outfile +} + +func getReportFile(options CommandLineOptions) *os.File { + reportFile, err := os.Create(*options.reportFilePtr) + if err != nil { + log.Fatalf("[getReportFile] error creating report file: %v", err) + } + log.Printf("[getReportFile] using report output file %s\n", *options.reportFilePtr) + return reportFile +} + +func sortedKeys(m map[string][]AccountEntry) []string { + var keys []string for k := range m { - keys[i] = k - i++ + keys = append(keys, k) } sort.Strings(keys) return keys } -func retrieveCookie(cookie string, readcookie bool, cookieDbFile string) (map[string]string, error) { +func retrieveCookie(cookie string, readCookie bool, cookieDbFile string) (map[string]string, error) { if cookie != "" { // cookie is given on the cli in CURL format log.Println("[retrieveCookie] retrieving cookies from cli") return deserializeCurlCookie(cookie) - } else if readcookie { + } else if readCookie { // cookie is to be read from Chrome's cookie database log.Println("[retrieveCookie] retrieving cookies from Chrome database") // wait for user to login @@ -210,48 +446,54 @@ func retrieveCookie(cookie string, readcookie bool, cookieDbFile string) (map[st scanner := bufio.NewScanner(os.Stdin) scanner.Scan() fmt.Println("Thanks! Now retrieving cookies from Chrome..") - cookiesCRH, err := kooky.ReadChromeCookies(cookieDbFile, "cloud.redhat.com", "", time.Time{}) + crhCookies, err := chrome.ReadCookies(cookieDbFile, kooky.Domain("cloud.redhat.com")) if err != nil { log.Fatalf("[retrieveCookie] error reading cookies from Chrome database: %v", err) return nil, err - } - cookiesRH, err := kooky.ReadChromeCookies(cookieDbFile, ".redhat.com", "", time.Time{}) + } + rhCookies, err := chrome.ReadCookies(cookieDbFile, kooky.DomainHasSuffix(".redhat.com")) if err != nil { log.Fatalf("[retrieveCookie] error reading cookies from Chrome database: %v", err) return nil, err - } - cookiesCRH = append(cookiesCRH, cookiesRH...) - return deserializeChromeCookie(cookiesCRH) - } + } + return deserializeChromeCookie(append(crhCookies, rhCookies...)) + } return nil, errors.New("[retrieveCookie] either --readcookie or --cookie= needs to be given") } -func pullAWS(awsPuller AWSPuller, reportfile *os.File, group string, account AccountEntry, csvData [][]string, month string, costType string) ([][]string, float64, error) { - log.Printf("[pullAWS] pulling AWS data for account %s", account.AccountID) - result, err := awsPuller.PullData(account.AccountID, month, costType) +func (a *AwsPuller) pullAwsAccount( + account AccountEntry, + group string, + month string, + costType string, + reportFile *os.File, +) (normalized *sheets.RowData, total float64, err error) { + result, err := a.PullData(account.AccountID, month, costType) if err != nil { - log.Fatalf("[pullAWS] error pulling data from AWS for account %s: %v", account.AccountID, err) - return csvData, 0, err - } - total, err := awsPuller.CheckResponseConsistency(account, result) + log.Fatalf("[pullAwsAccount] error pulling data from AWS for account %s: %v", account.AccountID, err) + } + total, err = a.CheckResponseConsistency(account, result) if err != nil { - log.Printf("[pullAWS] consistency check failed on response for account data %s: %v", account.AccountID, err) - writeReport(reportfile, account.AccountID + ": " + err.Error()) - } else { - log.Printf("[pullAWS] successful consistency check for data on account %s\n", account.AccountID) + log.Printf( + "[pullAwsAccount] consistency check failed on response for account data %s: %v", + account.AccountID, + err, + ) + writeReport(reportFile, account.AccountID+": "+err.Error()) } - normalized, err := awsPuller.NormalizeResponse(group, month, account.AccountID, result) + normalized, err = a.NormalizeResponse(group, month, account.AccountID, result) if err != nil { - log.Fatalf("[pullAWS] error normalizing data from AWS for account %s: %v", account.AccountID, err) - return csvData, 0, err - } - if csvData != nil { - csvData = appendCSVData(csvData, account.AccountID, normalized) + log.Fatalf("[pullAwsAccount] error normalizing data from AWS for account %s: %v", account.AccountID, err) } - return csvData, total, nil + return } -func pullCostManagement(cmPuller CMPuller, reportfile *os.File, account AccountEntry, csvData [][]string, month string) ([][]string, float64, error) { +func pullCostManagement( + cmPuller CmPuller, + reportFile *os.File, + account AccountEntry, + csvData [][]string, +) ([][]string, float64, error) { log.Printf("[pullCostManagement] pulling cost management data for account %s", account.AccountID) result, err := cmPuller.PullData(account.AccountID) if err != nil { @@ -265,8 +507,12 @@ func pullCostManagement(cmPuller CMPuller, reportfile *os.File, account AccountE } total, err := cmPuller.CheckResponseConsistency(account, parsed) if err != nil { - log.Printf("[pullCostManagement] error checking consistency of response for account data %s: %v", account.AccountID, err) - writeReport(reportfile, account.AccountID + " (CM): " + err.Error()) + log.Printf( + "[pullCostManagement] error checking consistency of response for account data %s: %v", + account.AccountID, + err, + ) + writeReport(reportFile, account.AccountID+" (CM): "+err.Error()) } else { log.Printf("[pullCostManagement] successful consistency check for data on account %s\n", account.AccountID) } @@ -275,19 +521,18 @@ func pullCostManagement(cmPuller CMPuller, reportfile *os.File, account AccountE log.Fatalf("[pullCostManagement] error normalizing data from service: %v", err) return csvData, 0, err } - if csvData != nil { - csvData = appendCSVData(csvData, account.AccountID, normalized) - } + log.Printf("[pullCostManagement] appended data for account %s\n", account.AccountID) + csvData = append(csvData, normalized) return csvData, total, nil } func deserializeCurlCookie(curlCookie string) (map[string]string, error) { deserialized := make(map[string]string) cookieElements := strings.Split(curlCookie, "; ") - for _, cookieStr := range(cookieElements) { + for _, cookieStr := range cookieElements { keyValue := strings.Split(cookieStr, "=") if len(keyValue) < 2 { - return nil, errors.New("[deserializecurlcookie] cookie not in correct format") + return nil, errors.New("[deserializeCurlCookie] cookie not in correct format") } deserialized[keyValue[0]] = keyValue[1] } @@ -302,74 +547,88 @@ func deserializeChromeCookie(chromeCookies []*kooky.Cookie) (map[string]string, return deserialized, nil } -func appendCSVHeader(csvData [][]string, group string) [][]string { - log.Printf("[appendcsvheader] appended header for group %s\n", group) - header := make([]string, 1) - header[0] = group - return append(csvData, header) -} - -func appendCSVData(csvData [][]string, account string, data []string) [][]string { - log.Printf("[appendcsvdata] appended data for account %s\n", account) - return append(csvData, data) -} - -func writeCSV(outfile *os.File, data [][]string) error { +func writeCsv(outfile *os.File, data [][]string) error { writer := csv.NewWriter(outfile) defer writer.Flush() for _, value := range data { err := writer.Write(value) if err != nil { - log.Printf("[writecsv] error writing csv data to file: %v ", err) + log.Printf("[writeCsv] error writing csv data to file: %v ", err) + return err + } + } + return nil +} + +func writeCsvFromSheet(outfile *os.File, data []*sheets.RowData) error { + writer := csv.NewWriter(outfile) + defer writer.Flush() + for _, row := range data { + rowData := make([]string, len(row.Values)) + for i, cell := range row.Values { + var cellData string + if cell.UserEnteredValue.StringValue != nil { + cellData = *cell.UserEnteredValue.StringValue + } else if cell.UserEnteredValue.NumberValue != nil { + cellData = fmt.Sprintf("%f", *cell.UserEnteredValue.NumberValue) + } else { + log.Fatalf("Unexpected sheet cell value: %v", cell.UserEnteredValue) + } + rowData[i] = cellData + } + err := writer.Write(rowData) + if err != nil { + log.Printf("[writeCsvFromSheet] error writing csv data to file: %v ", err) return err } } return nil } -func writeReport(outfile *os.File, data string) error { +func writeReport(outfile *os.File, data string) { _, err := outfile.WriteString(data + "\n") if err != nil { - log.Printf("[writereport] error writing report data to file: %v ", err) - return err + log.Printf("[writeReport] error writing report data to file: %v ", err) } - return nil } -func getAccountSetsFromFile(accountsFile string) (map[string][]AccountEntry, error) { - accounts := make(map[string][]AccountEntry) - yamlFile, err := ioutil.ReadFile(accountsFile) +func loadAccountsFile(accountsFileName string) (accountsFile AccountsFile, err error) { + yamlFile, err := os.ReadFile(accountsFileName) if err != nil { - log.Printf("[getaccountsets] error reading accounts file: %v ", err) - return nil, err + return accountsFile, fmt.Errorf("[loadAccountsFile] error loading accounts file: %v", err) } - err = yaml.Unmarshal(yamlFile, accounts) + accountsFile = AccountsFile{ + Configuration: make(map[string]map[string]string), + Providers: make(map[string]map[string][]AccountEntry), + } + err = yaml.Unmarshal(yamlFile, accountsFile) if err != nil { - log.Fatalf("[getaccountsets] error unmarshalling accounts file: %v", err) - return nil, err + return accountsFile, fmt.Errorf("[loadAccountsFile] error unmarshalling accounts file: %v", err) } // set category manually on all entries - for category, accountEntries := range accounts { - for _, accountEntry := range accountEntries { - accountEntry.Category = category + for _, group := range accountsFile.Providers { + for category, accountEntries := range group { + for _, accountEntry := range accountEntries { + accountEntry.Category = category + } } } - return accounts, nil + return } -func getAccountSetsFromAWS(awsPuller *AWSPuller) (map[string][]AccountEntry, error) { - log.Println("[main] initiating account metadata pull") - metadata, err := awsPuller.GetAWSAccountMetadata() +func getAccountSetsFromAws(awsPuller *AwsPuller) (map[string][]AccountEntry, error) { + log.Println("[getAccountSetsFromAws] initiating account metadata pull") + metadata, err := awsPuller.GetAwsAccountMetadata() if err != nil { - log.Fatalf("[main] error getting accounts list from metadata: %v", err) + log.Fatalf("[getAccountSetsFromAws] error getting accounts list from metadata: %v", err) } - log.Println("[main] processing account metadata pull") + log.Println("[getAccountSetsFromAws] processing account metadata pull") accounts := make(map[string][]AccountEntry) for accountID, accountMetadata := range metadata { - if category, ok := accountMetadata[AWSTagCostpullerCategory]; ok { - description := accountMetadata[AWSMetadataDescription] + if category, ok := accountMetadata[AwsTagCostpullerCategory]; ok { + description := accountMetadata[AwsMetadataDescription] log.Printf("tagged category (\"%s\") found for account %s (\"%s\")", category, accountID, description) - status := accountMetadata[AWSMetadataStatus] + status := accountMetadata[AwsMetadataStatus] if status == "ACTIVE" { if _, ok := accounts[category]; !ok { accounts[category] = []AccountEntry{} @@ -380,12 +639,37 @@ func getAccountSetsFromAWS(awsPuller *AWSPuller) (map[string][]AccountEntry, err Deviationpercent: 0, Category: category, Description: description, - }) + }) } } else { // account without category tag - log.Printf("ERRROR: account %s does not have an aws tag set for category (\"%s\")", accountID, accountMetadata[AWSMetadataDescription]) + log.Printf( + "ERROR: account %s does not have an aws tag set for category (\"%s\")", + accountID, + accountMetadata[AwsMetadataDescription], + ) } } - return accounts, nil -} \ No newline at end of file + return accounts, nil +} + +// closeFile is a helper function which allows closing a file to be deferred +// and which ignores any errors. +func closeFile(filename *os.File) { + _ = filename.Close() +} + +// getMapKeyValue is a generic helper function which fetches a value from the +// given key in the given map; if the key is not in the map, the program exits +// with an error citing the supplied section string. +func getMapKeyValue[V any]( + configMap map[string]V, + key string, + section string, +) (value V) { + if value, ok := configMap[key]; ok { + return value + } + log.Fatalf("Key %q is missing from the %q section of the configuration", key, section) + return +} diff --git a/gcloud_oauth.go b/gcloud_oauth.go new file mode 100644 index 0000000..0f7b02e --- /dev/null +++ b/gcloud_oauth.go @@ -0,0 +1,301 @@ +package main + +import ( + "context" + "crypto/sha256" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "log" + "net/http" + "net/url" + "os" + "path/filepath" + "regexp" + "time" + + "golang.org/x/oauth2" + "golang.org/x/oauth2/google" +) + +// defaultTokenCachePath is the path, relative to the platform's user cache +// directory, to the directory where cache files are stored. +const defaultTokenCachePath = "gcloud" + +// tokenFileName is the name of the file which is used to store the cached +// OAuth 2.0 access and refresh token values. +const tokenFileName = "costpuller_token.json" + +// getGoogleOAuthHttpClient accepts a mapping of configuration value strings +// and returns an HTTP client which can be used to make authorized Google API +// requests. The token is obtained either using values cached in a local file +// or by prompting the user to perform an authorization dialog; either way, the +// new token is written to the cache file before returning. +// +// The Google OAuth 2.0 Client configuration is constructed from a local +// credentials file (which can be downloaded from https://console.developers.google.com, +// under "Credentials"). It is located using the default mechanisms (e.g., in +// ${HOME}/.config/gcloud/application_default_credentials.json). (Currently, +// the scope of the authorization is limited to the Google Sheets APIs.) +func getGoogleOAuthHttpClient(oauthConfigMap map[string]string) *http.Client { + ctx := context.Background() + + credObj, err := google.FindDefaultCredentials(ctx, "https://www.googleapis.com/auth/spreadsheets") + if err != nil { + log.Fatalf("Unable to read OAuth client credentials file: %v", err) + } + + config, err := google.ConfigFromJSON(credObj.JSON, "https://www.googleapis.com/auth/spreadsheets") + if err != nil { + log.Fatalf("Unable to construct a client configuration: %v", err) + } + + token, tokenCachePath := getToken(oauthConfigMap, config, ctx) + cacheToken(token, tokenCachePath) + + return config.Client(ctx, token) +} + +// getToken is a helper function which extracts configuration information from +// the supplied mapping and returns either a cached token, if available, or a +// new token. +func getToken( + oauthConfigMap map[string]string, + config *oauth2.Config, + ctx context.Context, +) (token *oauth2.Token, tokenCachePath string) { + var tokenCacheFile *os.File + tokenCachePath, err := getCacheFileName(oauthConfigMap["tokenCachePath"]) + if err == nil { + tokenCacheFile, err = os.Open(tokenCachePath) + } + if err == nil { + token = getCachedToken(config, tokenCacheFile, ctx) + closeFile(tokenCacheFile) + } else if errors.Is(err, os.ErrNotExist) { + token = getNewToken(config, oauthConfigMap["port"], ctx) + } else { + log.Fatalf("Unexpected error accessing the token cache file, %q: %v", tokenCachePath, err) + } + return +} + +// cacheToken is a helper function which accepts a token and a file path and +// stores the token in the indicated file. The contents of the file are +// replaced with the new value. If the path is blank, the function prints a +// message and returns; other errors result in exiting the process. +func cacheToken(token *oauth2.Token, tokenCachePath string) { + if tokenCachePath == "" { + log.Println("The token will not be cached.") + } else { + newTokenCacheFile, err := os.OpenFile(tokenCachePath, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0600) + if err == nil { + log.Printf("Caching oauth token in %q.", tokenCachePath) + err = json.NewEncoder(newTokenCacheFile).Encode(token) + closeFile(newTokenCacheFile) + } + if err != nil { + log.Printf("Unable to cache oauth token: %v", err) + } + } +} + +// getCacheFileName accepts a file path to the directory containing the token +// cache file and returns an absolute path to the cached token file or an +// error. If the input path is an empty string, the default path is used; if +// the path is relative, it is prefixed with the platform's user configuration +// directory. The token file name is appended to the path and the result is +// returned. +func getCacheFileName(tokenCachePath string) (string, error) { + if tokenCachePath == "" { + tokenCachePath = defaultTokenCachePath + } + if tokenCachePath[0] != '/' { + cacheDir, err := os.UserCacheDir() + if err != nil { + log.Printf("unable to determine cache directory: %v", err) + return "", fmt.Errorf("%w", os.ErrNotExist) + } + tokenCachePath = filepath.Join(cacheDir, tokenCachePath) + if err := os.MkdirAll(tokenCachePath, 0700); err != nil { + log.Printf("unable to create user cache dir, %q: %v", cacheDir, err) + return "", fmt.Errorf("%w", os.ErrNotExist) + } + } + return filepath.Join(tokenCachePath, tokenFileName), nil +} + +// getCachedToken is a helper function which reads a cached token from the +// provided file, refreshes it using the provided configuration and context, +// and returns the resulting token. +func getCachedToken(config *oauth2.Config, cacheFile *os.File, ctx context.Context) *oauth2.Token { + token := &oauth2.Token{} + err := json.NewDecoder(cacheFile).Decode(token) + if err != nil { + log.Fatalf("Unable to parse cached OAuth tokens, %q: %v", cacheFile.Name(), err) + } + + token, err = config.TokenSource(ctx, token).Token() + if err != nil { + log.Fatalf("Unable to refresh the cached OAuth tokens: %v", err) + } + + return token +} + +// getNewToken is a helper function which prompts the user to use their browser +// to request a new token, obtains the access code when the request is +// redirected to the local listener, exchanges the access code for an access +// token and a refresh token, and returns the token-pair. The supplied +// configuration is used to access the OAuth 2.0 client configuration to +// generate the access request URL; the redirect URL is modified to include +// a custom port (otherwise, it would default to port 80, which is not +// generally available); and, a random number ("state") is included in the +// request and checked in the redirect to prevent man-in-the-middle attacks. +// After prompting the user, a local listener for the redirect request is +// started, and execution waits for the redirected request which includes the +// access code in the request query parameters. +func getNewToken(config *oauth2.Config, listenerPort string, ctx context.Context) *oauth2.Token { + stateToken := getStateToken() + if listenerPort == "" { + listenerPort = "35355" // Arbitrary value + } + config.RedirectURL += ":" + listenerPort + authURL := config.AuthCodeURL(stateToken, oauth2.AccessTypeOffline) + fmt.Printf("\nGo to the following link in your browser to authorize access:\n%v\n\n", authURL) + + // Listen for the redirect request, then extract the authorization code + // from the resulting query params. + queryParams := redirectListener(config.RedirectURL) + authCode := getAuthCode(queryParams, stateToken) + + // Exchange the authorization code for an access token and refresh token. + token, err := config.Exchange(ctx, authCode) + if err != nil { + log.Fatalf("Unable to retrieve access token: %v", err) + } + return token +} + +// getStateToken creates a random state token which is used to validate the +// OAuth redirect request. The token is the base64-encoded SHA256 hash of the +// current time as a string. +func getStateToken() string { + h := sha256.New() + h.Write([]byte(time.Now().Format("20060102150405000000"))) + return base64.StdEncoding.EncodeToString(h.Sum(nil)) +} + +// getAuthCode validates the result of the redirect from the user's +// authorization request, and returns the access code if one is received; +// otherwise it exits the process with a failure. +func getAuthCode(authResp url.Values, stateToken string) string { + if authResp.Get("state") != stateToken { + log.Fatalf( + "Error in authorization state, expected %q, got %q", + stateToken, + authResp.Get("state"), + ) + } + if authResp.Get("error") != "" { + log.Fatalf("Error returned from authorization: %s", authResp.Get("error")) + } + authCode := authResp.Get("code") + if authCode == "" { + log.Fatalf("No authorization code received.") + } + return authCode +} + +// redirectListener is a helper function used in the creation of the Google API +// client. It sets up a micro-webserver which listens for a single request to +// the provided URL. Errors parsing the redirect URL input or starting the +// micro-webserver are logged with Fatalf() which exits the process. +// +// When the request is received, the request is acknowledged, the webserver is +// shut down, and the query parameters of the request (presumably the state +// token and the access code; or an error) are returned. The request (in the +// user's browser) looks something like this: +// +// http://localhost/?state=&code=&scope= +func redirectListener(urlString string) url.Values { + // This variable is set by the request handler (it is included in the + // function's closure) and returned after the micro-webserver exits. + var queryParams url.Values + + // Configure the micro-webserver, add a handler to it for the default + // route, and start the listener which will serve requests until the + // server is shut down. + mux := http.NewServeMux() + server := http.Server{Addr: getListenAddress(urlString), Handler: mux} + mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + queryParams = r.URL.Query() + handleRedirectResponse(w, queryParams) + // Request the server shutdown in a separate goroutine to allow it to + // wait for this request to finish processing. + go requestShutdown(&server) + }) + + // Run the webserver, listening for and dispatching requests, until + // shutdown is requested. + if err := server.ListenAndServe(); err != nil { + if !errors.Is(err, http.ErrServerClosed) { + log.Fatalf("Error running redirect listener: %v", err) + } + } + + return queryParams +} + +// handleRedirectResponse is a helper function which evaluates the redirect +// query parameters and sends an appropriate response to the request. +func handleRedirectResponse(w http.ResponseWriter, queryParams url.Values) { + msg := `

` + if queryParams.Get("code") == "" { + msg += "Failure -- no access code received!

" + if queryParams.Get("error") != "" { + msg += "

Error: " + queryParams.Get("error") + "

" + } + } else { + msg += "Success! Access code received." + } + msg += "

You may close this browser window." + _, err := fmt.Fprint(w, msg) + if err != nil { + log.Printf("Error writing response to redirect request: %v", err) + } +} + +// requestShutdown is a helper function which requests the server to shut down, +// packaged as a separate function to make it easy to run as a goroutine. +func requestShutdown(server *http.Server) { + err := server.Shutdown(context.Background()) + if err != nil { + log.Fatalf("Error shutting down redirect listener: %v", err) + } +} + +// RedirectUrlPattern matches a host (e.g., "localhost" or a FQDN) with an +// optional "http" schema and an optional port. This is the location provided +// in the OAuth 2.0 client configuration where the authorization flow redirects +// the request after it has been granted or denied. The schema, if any is +// ignored; path specifications are not supported -- only host (and optionally +// port) should be provided. The host must resolve to a NIC on the machine +// where this program is being run. +var RedirectUrlPattern = regexp.MustCompile(`^(?:http://)?([^:/]+)(:[0-9]{1,5})$`) + +// getListenAddress validates the redirect URL, strips the schema if present, +// sets the address to the host, appends the port if present, and returns the +// result. +func getListenAddress(urlString string) string { + matches := RedirectUrlPattern.FindStringSubmatch(urlString) + if matches == nil { + log.Fatalf("Could not parse redirect URL: %s", urlString) + } + address := matches[1] + if matches[2] != "" { + address += matches[2] + } + return address +} diff --git a/go.mod b/go.mod index 9d8f303..bffc85d 100644 --- a/go.mod +++ b/go.mod @@ -1,11 +1,46 @@ -module github.com/michaelkleinhenz/costpuller +module costpuller -go 1.13 +go 1.22 + +toolchain go1.22.5 + +require ( + github.com/aws/aws-sdk-go v1.55.3 + github.com/browserutils/kooky v0.2.2 + github.com/jinzhu/now v1.1.5 + golang.org/x/oauth2 v0.21.0 + google.golang.org/api v0.189.0 + gopkg.in/yaml.v2 v2.4.0 +) require ( - github.com/aws/aws-sdk-go v1.29.23 - github.com/go-delve/delve v1.4.1 // indirect - github.com/jinzhu/now v1.1.1 - github.com/zellyn/kooky v0.0.0-20200206144811-607d4ccbb896 - gopkg.in/yaml.v2 v2.2.8 + cloud.google.com/go/auth v0.7.2 // indirect + cloud.google.com/go/auth/oauth2adapt v0.2.3 // indirect + cloud.google.com/go/compute/metadata v0.5.0 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/go-logr/logr v1.4.2 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/go-sqlite/sqlite3 v0.0.0-20180313105335-53dd8e640ee7 // indirect + github.com/godbus/dbus/v5 v5.1.0 // indirect + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/gonuts/binary v0.2.0 // indirect + github.com/google/s2a-go v0.1.8 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect + github.com/googleapis/gax-go/v2 v2.13.0 // indirect + github.com/jmespath/go-jmespath v0.4.0 // indirect + github.com/keybase/go-keychain v0.0.0-20231219164618-57a3676c3af6 // indirect + github.com/zalando/go-keyring v0.2.5 // indirect + go.opencensus.io v0.24.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0 // indirect + go.opentelemetry.io/otel v1.28.0 // indirect + go.opentelemetry.io/otel/metric v1.28.0 // indirect + go.opentelemetry.io/otel/trace v1.28.0 // indirect + golang.org/x/crypto v0.25.0 // indirect + golang.org/x/net v0.27.0 // indirect + golang.org/x/sys v0.22.0 // indirect + golang.org/x/text v0.16.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240725223205-93522f1f2a9f // indirect + google.golang.org/grpc v1.65.0 // indirect + google.golang.org/protobuf v1.34.2 // indirect ) diff --git a/go.sum b/go.sum index 8d5c093..c43e242 100644 --- a/go.sum +++ b/go.sum @@ -1,80 +1,188 @@ -github.com/aws/aws-sdk-go v1.29.23 h1:wtiGLOzxAP755OfuVTDIy/NbUIYEDxbIbBEDfNhUpeU= -github.com/aws/aws-sdk-go v1.29.23/go.mod h1:1KvfttTE3SPKMpo8g2c6jL3ZKfXtFvKscTgahTma5Xg= -github.com/cosiner/argv v0.0.0-20170225145430-13bacc38a0a5 h1:rIXlvz2IWiupMFlC45cZCXZFvKX/ExBcSLrDy2G0Lp8= -github.com/cosiner/argv v0.0.0-20170225145430-13bacc38a0a5/go.mod h1:p/NrK5tF6ICIly4qwEDsf6VDirFiWWz0FenfYBwJaKQ= -github.com/cpuguy83/go-md2man v1.0.10 h1:BSKMNlYxDvnunlTymqtgONjNnaRV1sTpcovwwjF22jk= -github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= -github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/danieljoos/wincred v1.0.2/go.mod h1:SnuYRW9lp1oJrZX/dXJqr0cPK5gYXqx3EJbmjhLdK9U= +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go/auth v0.7.2 h1:uiha352VrCDMXg+yoBtaD0tUF4Kv9vrtrWPYXwutnDE= +cloud.google.com/go/auth v0.7.2/go.mod h1:VEc4p5NNxycWQTMQEDQF0bd6aTMb6VgYDXEwiJJQAbs= +cloud.google.com/go/auth/oauth2adapt v0.2.3 h1:MlxF+Pd3OmSudg/b1yZ5lJwoXCEaeedAguodky1PcKI= +cloud.google.com/go/auth/oauth2adapt v0.2.3/go.mod h1:tMQXOfZzFuNuUxOypHlQEXgdfX5cuhwU+ffUuXRJE8I= +cloud.google.com/go/compute/metadata v0.5.0 h1:Zr0eK8JbFv6+Wi4ilXAR8FJ3wyNdpxHKJNPos6LTZOY= +cloud.google.com/go/compute/metadata v0.5.0/go.mod h1:aHnloV2TPI38yx4s9+wAZhHykWvVCfu7hQbF+9CWoiY= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/Velocidex/json v0.0.0-20220224052537-92f3c0326e5a h1:AeXPUzhU0yhID/v5JJEIkjaE85ASe+Vh4Kuv1RSLL+4= +github.com/Velocidex/json v0.0.0-20220224052537-92f3c0326e5a/go.mod h1:ukJBuruT9b24pdgZwWDvOaCYHeS03B7oQPCUWh25bwM= +github.com/Velocidex/ordereddict v0.0.0-20230909174157-2aa49cc5d11d h1:fn372EqKyazBxYUP5HPpBi3jId4MXuppEypEALGfvEk= +github.com/Velocidex/ordereddict v0.0.0-20230909174157-2aa49cc5d11d/go.mod h1:+MqO5UMBemyFSm+yRXslbpFTwPUDhFHUf7HPV92twg4= +github.com/Velocidex/yaml/v2 v2.2.8 h1:GUrSy4SBJ6RjGt43k6MeBKtw2z/27gh4A3hfFmFY3No= +github.com/Velocidex/yaml/v2 v2.2.8/go.mod h1:PlXIg/Pxmoja48C1vMHo7C5pauAZvLq/UEPOQ3DsjS4= +github.com/aws/aws-sdk-go v1.55.3 h1:0B5hOX+mIx7I5XPOrjrHlKSDQV/+ypFZpIHOx5LOk3E= +github.com/aws/aws-sdk-go v1.55.3/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU= +github.com/browserutils/kooky v0.2.2 h1:uLKlE294eXudGEAt/NjOrL5Nzbi57ZtkuWwKZ1hT13I= +github.com/browserutils/kooky v0.2.2/go.mod h1:Ls7BAtUgrzzi5AfD1T4CqDu7mhHAaGMwCx6kH2nnjHI= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/go-delve/delve v1.4.1 h1:kZs0umEv+VKnK84kY9/ZXWrakdLTeRTyYjFdgLelZCQ= -github.com/go-delve/delve v1.4.1/go.mod h1:vmy6iObn7zg8FQ5KOCIe6TruMNsqpoZO8uMiRea+97k= -github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= +github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-sqlite/sqlite3 v0.0.0-20180313105335-53dd8e640ee7 h1:ow5vK9Q/DSKkxbEIJHBST6g+buBDwdaDIyk1dGGwpQo= github.com/go-sqlite/sqlite3 v0.0.0-20180313105335-53dd8e640ee7/go.mod h1:JxSQ+SvsjFb+p8Y+bn+GhTkiMfKVGBD0fq43ms2xw04= -github.com/godbus/dbus v4.1.0+incompatible h1:WqqLRTsQic3apZUK9qC5sGNfXthmPXzUZ7nQPrNITa4= -github.com/godbus/dbus v4.1.0+incompatible/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw= +github.com/godbus/dbus/v5 v5.1.0 h1:4KLkAxT3aOY8Li4FRJe/KvhoNFFxo0m6fNuFUO8QJUk= +github.com/godbus/dbus/v5 v5.1.0/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/gonuts/binary v0.2.0 h1:caITwMWAoQWlL0RNvv2lTU/AHqAJlVuu6nZmNgfbKW4= github.com/gonuts/binary v0.2.0/go.mod h1:kM+CtBrCGDSKdv8WXTuCUsw+loiy8f/QEI8YCCC0M/E= -github.com/google/go-dap v0.2.0 h1:whjIGQRumwbR40qRU7CEKuFLmePUUc2s4Nt9DoXXxWk= -github.com/google/go-dap v0.2.0/go.mod h1:5q8aYQFnHOAZEMP+6vmq25HKYAEwE+LF5yh7JKrrhSQ= -github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= -github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= -github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= -github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/jinzhu/now v1.1.1 h1:g39TucaRWyV3dwDO++eEc6qf8TVIQ/Da48WmqjZ3i7E= -github.com/jinzhu/now v1.1.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= -github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM= -github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= -github.com/keybase/go-keychain v0.0.0-20191220220820-f65a47cbe0b1/go.mod h1:JJNrCn9otv/2QP4D7SMJBgaleKpOf66PnW6F5WGNRIc= -github.com/konsorten/go-windows-terminal-sequences v1.0.3 h1:CE8S1cTafDpPvMhIxNJKvHsGVBgn1xWYf1NbHQhywc8= -github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/mattn/go-colorable v0.0.0-20170327083344-ded68f7a9561 h1:isR/L+BIZ+rqODWYR/f526ygrBMGKZYFhaaFRDGvuZ8= -github.com/mattn/go-colorable v0.0.0-20170327083344-ded68f7a9561/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= -github.com/mattn/go-isatty v0.0.3 h1:ns/ykhmWi7G9O+8a448SecJU3nSMBXJfqQkl0upE1jI= -github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= -github.com/peterh/liner v0.0.0-20170317030525-88609521dc4b h1:8uaXtUkxiy+T/zdLWuxa/PG4so0TPZDZfafFNNSaptE= -github.com/peterh/liner v0.0.0-20170317030525-88609521dc4b/go.mod h1:xIteQHvHuaLYG9IFj6mSxM0fCKrs34IrEQUhOYuGPHc= -github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/s2a-go v0.1.8 h1:zZDs9gcbt9ZPLV0ndSyQk6Kacx2g/X+SKYovpnz3SMM= +github.com/google/s2a-go v0.1.8/go.mod h1:6iNWHTpQ+nfNRN5E00MSdfDwVesa8hhS32PhPO8deJA= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/enterprise-certificate-proxy v0.3.2 h1:Vie5ybvEvT75RniqhfFxPRy3Bf7vr3h0cechB90XaQs= +github.com/googleapis/enterprise-certificate-proxy v0.3.2/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0= +github.com/googleapis/gax-go/v2 v2.13.0 h1:yitjD5f7jQHhyDsnhKEBU52NdvvdSeGzlAnDPT0hH1s= +github.com/googleapis/gax-go/v2 v2.13.0/go.mod h1:Z/fvTZXF8/uw7Xu5GuslPw+bplx6SS338j1Is2S+B7A= +github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= +github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= +github.com/keybase/go-keychain v0.0.0-20231219164618-57a3676c3af6 h1:IsMZxCuZqKuao2vNdfD82fjjgPLfyHLpR41Z88viRWs= +github.com/keybase/go-keychain v0.0.0-20231219164618-57a3676c3af6/go.mod h1:3VeWNIJaW+O5xpRQbPp0Ybqu1vJd/pm7s2F473HRrkw= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/russross/blackfriday v1.5.2 h1:HyvC0ARfnZBqnXwABFeSZHpKvJHJJfPz81GNueLj0oo= -github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= -github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I= -github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= -github.com/spf13/cobra v0.0.0-20170417170307-b6cb39589372 h1:eRfW1vRS4th8IX2iQeyqQ8cOUNOySvAYJ0IUvTXGoYA= -github.com/spf13/cobra v0.0.0-20170417170307-b6cb39589372/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= -github.com/spf13/pflag v0.0.0-20170417173400-9e4c21054fa1 h1:7bozMfSdo41n2NOc0GsVTTVUiA+Ncaj6pXNpm4UHKys= -github.com/spf13/pflag v0.0.0-20170417173400-9e4c21054fa1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/zalando/go-keyring v0.0.0-20200121091418-667557018717 h1:3M/uUZajYn/082wzUajekePxpUAZhMTfXvI9R+26SJ0= -github.com/zalando/go-keyring v0.0.0-20200121091418-667557018717/go.mod h1:RaxNwUITJaHVdQ0VC7pELPZ3tOWn13nr0gZMZEhpVU0= -github.com/zellyn/kooky v0.0.0-20200206144811-607d4ccbb896 h1:qiQQO+IEgIGuqYXkgxpagp9lXL5u9o6u9SvHHUXirR4= -github.com/zellyn/kooky v0.0.0-20200206144811-607d4ccbb896/go.mod h1:KBLJ6p0HNW5ffhMF9uIUAAp3HZFefBUjxxT0izMsGcI= -go.starlark.net v0.0.0-20190702223751-32f345186213 h1:lkYv5AKwvvduv5XWP6szk/bvvgO6aDeUujhZQXIFTes= -go.starlark.net v0.0.0-20190702223751-32f345186213/go.mod h1:c1/X6cHgvdXj6pUlmWKMkuqRnW4K8x2vwt6JAaaircg= -golang.org/x/arch v0.0.0-20190927153633-4e8777c89be4 h1:QlVATYS7JBoZMVaf+cNjb90WD/beKVHnIxFKT4QaHVI= -golang.org/x/arch v0.0.0-20190927153633-4e8777c89be4/go.mod h1:flIaEI6LNU6xOCD5PaJvn9wGP0agmIOqjrtsKGRguv4= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/zalando/go-keyring v0.2.5 h1:Bc2HHpjALryKD62ppdEzaFG6VxL6Bc+5v0LYpN8Lba8= +github.com/zalando/go-keyring v0.2.5/go.mod h1:HL4k+OXQfJUWaMnqyuSOc0drfGPX2b51Du6K+MRgZMk= +go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= +go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0 h1:4K4tsIXefpVJtvA/8srF4V4y0akAoPHkIslgAkjixJA= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0/go.mod h1:jjdQuTGVsXV4vSs+CJ2qYDeDPf9yIJV23qlIzBm73Vg= +go.opentelemetry.io/otel v1.28.0 h1:/SqNcYk+idO0CxKEUOtKQClMK/MimZihKYMruSMViUo= +go.opentelemetry.io/otel v1.28.0/go.mod h1:q68ijF8Fc8CnMHKyzqL6akLO46ePnjkgfIMIjUIX9z4= +go.opentelemetry.io/otel/metric v1.28.0 h1:f0HGvSl1KRAU1DLgLGFjrwVyismPlnuU6JD6bOeuA5Q= +go.opentelemetry.io/otel/metric v1.28.0/go.mod h1:Fb1eVBFZmLVTMb6PPohq3TO9IIhUisDsbJoL/+uQW4s= +go.opentelemetry.io/otel/trace v1.28.0 h1:GhQ9cUuQGmNDd5BTCP2dAvv75RdMxEfTmYejp+lkx9g= +go.opentelemetry.io/otel/trace v1.28.0/go.mod h1:jPyXzNPg6da9+38HEwElrQiHlVMTnVfM3/yv2OlIHaI= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20200204104054-c9f3fb736b72 h1:+ELyKg6m8UBf0nPFSqD0mi7zUfwPyXo23HNjMnXPz7w= -golang.org/x/crypto v0.0.0-20200204104054-c9f3fb736b72/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.25.0 h1:ypSNr+bnYL2YhwoMt2zPxHFmbAN1KZs/njMG3hxUp30= +golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys= +golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.21.0 h1:tsimM75w1tF/uws5rbeHzIWxEqElMehnc+iW793zsZs= +golang.org/x/oauth2 v0.21.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb h1:fgwFCsaw9buMuxNd6+DQfAuSFqbNiQZpcgJQAgJsK6k= -golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= +golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/tools v0.0.0-20191127201027-ecd32218bd7f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= +golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/api v0.189.0 h1:equMo30LypAkdkLMBqfeIqtyAnlyig1JSZArl4XPwdI= +google.golang.org/api v0.189.0/go.mod h1:FLWGJKb0hb+pU2j+rJqwbnsF+ym+fQs73rbJ+KAUgy8= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20240722135656-d784300faade h1:lKFsS7wpngDgSCeFn7MoLy+wBDQZ1UQIJD4UNM1Qvkg= +google.golang.org/genproto/googleapis/api v0.0.0-20240711142825-46eb208f015d h1:kHjw/5UfflP/L5EbledDrcG4C2597RtymmGRZvHiCuY= +google.golang.org/genproto/googleapis/api v0.0.0-20240711142825-46eb208f015d/go.mod h1:mw8MG/Qz5wfgYr6VqVCiZcHe/GJEfI+oGGDCohaVgB0= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240725223205-93522f1f2a9f h1:RARaIm8pxYuxyNPbBQf5igT7XdOyCNtat1qAT2ZxjU4= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240725223205-93522f1f2a9f/go.mod h1:Ue6ibwXGpU+dqIcODieyLOcgj7z8+IcskoNIgZxtrFY= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= +google.golang.org/grpc v1.65.0 h1:bs/cUb4lp1G5iImFFd3u5ixQzweKizoZJAwBNLR42lc= +google.golang.org/grpc v1.65.0/go.mod h1:WgYC2ypjlB0EiQi6wdKixMqukr6lBc0Vo+oOgjrM5ZQ= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +www.velocidex.com/golang/go-ese v0.2.0 h1:8/hzEMupfqEF0oMi1/EzsMN1xLN0GBFcB3GqxqRnb9s= +www.velocidex.com/golang/go-ese v0.2.0/go.mod h1:6fC9T6UGLbM7icuA0ugomU5HbFC5XA5I30zlWtZT8YE= diff --git a/gsheets.go b/gsheets.go new file mode 100644 index 0000000..ab7339c --- /dev/null +++ b/gsheets.go @@ -0,0 +1,220 @@ +package main + +import ( + "context" + "google.golang.org/api/option" + "google.golang.org/api/sheets/v4" + "log" + "net/http" + "time" +) + +// postToGSheet creates a new sheet in a Google Sheets spreadsheet and loads it +// with the specified data. Requests are made to the Google API using the +// specified HTTP client which has already been authenticated and authorized. +// The new sheet name is constructed based on the reference time passed in the +// last parameter. Details such as the spreadsheet ID and sheet names are found +// in the configuration map. +func postToGSheet(sheetData []*sheets.RowData, client *http.Client, configMap map[string]string, ref time.Time) { + srv, err := sheets.NewService(context.Background(), option.WithHTTPClient(client)) + if err != nil { + log.Fatalf("Unable to create Google Sheets client: %v", err) + } + + // Construct the name for the raw data sheet using the template-name from + // the configuration as a format specifier for time.Format() + // (see https://pkg.go.dev/time#Layout). Format fields (represented by + // strings of digits) are replaced with portions of the reference time + // value while non-digits are copied literally, so, if the template-name is + // "Raw Data 01/2006" and the reference time is in August 2024, the result + // will be "Raw Data 08/2024". + newSheetName := ref.Format(getMapKeyValue(configMap, "sheetNameTemplate", "gsheet")) + + spreadsheetId := getMapKeyValue(configMap, "spreadsheetId", "gsheet") + log.Println("Fetching Spreadsheet information") + sheetObject, err := srv.Spreadsheets. + Get(spreadsheetId). + Fields("sheets/properties(gridProperties(columnCount,rowCount),sheetId,title)", "spreadsheetId"). + Do() + if err != nil { + log.Fatalf("Error retrieving spreadsheet: %v", err) + } + + newDataRef := getUpdateLocation(srv, sheetObject, newSheetName, len(sheetData[0].Values), configMap) + + mainSheetName := getMapKeyValue(configMap, "mainSheetName", "gsheet") + mainSheetID, msIndex := getSheetIDFromName(sheetObject, mainSheetName) + if msIndex < 0 { + log.Fatalf("Error updating spreadsheet sheet: main sheet %q not found", mainSheetName) + } + + cells, err := srv.Spreadsheets.Values.Get(spreadsheetId, "'"+mainSheetName+"'!A1:ZZZ9999").Do() + if err != nil { + log.Fatalf("Error fetching main sheet (%q) values: %v", mainSheetID, err) + } + mainSheetRef := getNewSheetReference(cells, mainSheetID, newSheetName, len(sheetData)) + if mainSheetRef == nil { + log.Fatalf("No reference to %q found in main sheet (%q)", newSheetName, mainSheetName) + } + loadNewData(srv, spreadsheetId, sheetData, newDataRef, mainSheetRef) +} + +// getUpdateLocation is a helper function which returns the GridRange to +// receive the new data. This includes looking up the existing sheet or +// creating a new one. +func getUpdateLocation( + srv *sheets.Service, + sheetObject *sheets.Spreadsheet, + newSheetName string, + newColumnCount int, + configMap map[string]string, +) (newDataRef *sheets.GridRange) { + _, newIndex := getSheetIDFromName(sheetObject, newSheetName) + if newIndex < 0 { + templateSheetName := getMapKeyValue(configMap, "templateSheetName", "gsheet") + srcID, srcIndex := getSheetIDFromName(sheetObject, templateSheetName) + if srcIndex < 0 { + log.Fatalf("Error updating spreadsheet sheet: template sheet %q not found", templateSheetName) + } + + srcColumnCount := sheetObject.Sheets[srcIndex].Properties.GridProperties.ColumnCount + if int64(newColumnCount) >= srcColumnCount { + log.Fatalf( + "Unexpected column count for data: received %d; expected fewer than %d", + srcColumnCount, + newColumnCount, + ) + } + + log.Printf("Adding new sheet %q", newSheetName) + spreadsheetId := sheetObject.SpreadsheetId + newDataRef = createNewSheet(srv, spreadsheetId, newSheetName, srcID, int64(len(sheetObject.Sheets))) + } else { + log.Printf("Warning: overwriting sheet %q", newSheetName) + newDataRef = getDataGridRange(sheetObject.Sheets[newIndex].Properties) + } + return newDataRef +} + +// loadNewData updates the data cells (avoiding the header row and the totals +// column) in the indicated sheet of the indicated spreadsheet from the +// provided RowData using the provided service client; it then copies a range +// of cells new sheet with the new data, and then poke the main sheet +// to get it to update its references to the new sheet. +func loadNewData( + srv *sheets.Service, + spreadsheetId string, + sheetData []*sheets.RowData, + newSheetRef *sheets.GridRange, + mainSheetRef *sheets.GridRange, +) { + _, err := srv.Spreadsheets.BatchUpdate(spreadsheetId, &sheets.BatchUpdateSpreadsheetRequest{ + Requests: []*sheets.Request{ + { + UpdateCells: &sheets.UpdateCellsRequest{ + Fields: "userEnteredValue", + Range: newSheetRef, + Rows: sheetData, + }, + }, + { + CopyPaste: &sheets.CopyPasteRequest{ + Destination: mainSheetRef, + PasteOrientation: "NORMAL", + PasteType: "PASTE_NORMAL", + Source: mainSheetRef, + }, + }, + }, + }).Do() + if err != nil { + log.Fatalf("Error updating sheet: %v", err) + } +} + +// createNewSheet creates a new sheet in the provided spreadsheet using the +// provided service client by duplicating the provided source sheet and +// inserting it into the spreadsheet at the indicated position with the +// provided name; it then returns the address of a GridRange describing where +// to place the new data (avoiding the header row). +func createNewSheet(srv *sheets.Service, spreadsheetId string, newSheetName string, srcID int64, position int64) *sheets.GridRange { + buResp, err := srv.Spreadsheets.BatchUpdate(spreadsheetId, &sheets.BatchUpdateSpreadsheetRequest{ + Requests: []*sheets.Request{ + { + DuplicateSheet: &sheets.DuplicateSheetRequest{ + NewSheetName: newSheetName, + SourceSheetId: srcID, + InsertSheetIndex: position, + }, + }, + }, + }).Do() + if err != nil { + log.Fatalf("Error duplicating sheet: %v", err) + } + + props := buResp.Replies[0].DuplicateSheet.Properties + return getDataGridRange(props) +} + +// getGridRange is a helper function which produces a GridRange describing the +// contents of a sheet, minus the header row and last column, given the sheet's +// properties object. +func getDataGridRange(props *sheets.SheetProperties) *sheets.GridRange { + cc := props.GridProperties.ColumnCount + rc := props.GridProperties.RowCount + id := props.SheetId + + return &sheets.GridRange{ + EndColumnIndex: cc - 1, // Skip "Totals" column + EndRowIndex: rc, + SheetId: id, + StartColumnIndex: 0, + StartRowIndex: 1, // Skip header row + } +} + +// getNewSheetReference returns a pointer to a GridRange which describes the +// cells in the provided main sheet which (indirectly) refer to the indicated +// new sheet. This is done by locating the cell in the provided ValueRange +// which refers to the provided new sheet by name; we assume the indirect +// references are in the same column starting in the row below the matching +// cell and that there will be the provided number of rows. +func getNewSheetReference( + cells *sheets.ValueRange, + mainSheetID int64, + newSheetName string, + rowCount int, +) *sheets.GridRange { + for r, row := range cells.Values { + for c, cell := range row { + if str, ok := cell.(string); ok { + if str == newSheetName { + msColumn := int64(c) + msRow := int64(r + 1) + // Indices are zero-based, starts are inclusive, ends are exclusive. + return &sheets.GridRange{ + EndColumnIndex: msColumn + 1, + EndRowIndex: msRow + int64(rowCount) + 1, + SheetId: mainSheetID, + StartColumnIndex: msColumn, + StartRowIndex: msRow, + } + } + } + } + } + return nil +} + +// getSheetIDFromName is a helper function which returns the sheet ID for the +// sheet (tab) with the given name in the specified spreadsheet. Returns a +// boolean indicating if the name was not found. +func getSheetIDFromName(sheetObject *sheets.Spreadsheet, sheetName string) (int64, int) { + for index, sheet := range sheetObject.Sheets { + if sheet.Properties.Title == sheetName { + return sheet.Properties.SheetId, index + } + } + return -1, -1 +}