From 5e36f14a5a9fed670008f7d1eddb8351638ad7d2 Mon Sep 17 00:00:00 2001 From: Alomerry Wu Date: Tue, 30 Apr 2024 23:37:28 +0800 Subject: [PATCH] feat: add mysql backup tool --- README.md | 2 +- go.mod | 22 +- go.sum | 108 +++++---- main.go | 7 - modules/pusher/README_ZH.md | 60 ----- modules/pusher/component/oss/kodo/kodo.go | 115 ---------- modules/pusher/component/oss/kodo/request.go | 6 - modules/pusher/component/oss/oss.go | 65 ------ modules/pusher/config/example.toml | 29 --- modules/pusher/main.go | 25 -- modules/pusher/modes/client.go | 62 ----- modules/pusher/modes/config.go | 51 ----- modules/pusher/modes/pusher/pusher.go | 227 ------------------- modules/pusher/modes/syncer/syncer.go | 50 ---- modules/pusher/modes/task.go | 9 - modules/pusher/share/errors.go | 7 - modules/pusher/share/interface.go | 9 - modules/pusher/share/share.go | 31 --- modules/pusher/utils/utils.go | 27 --- modules/pusher/utils/watcher.go | 41 ---- static/constant/database.go | 25 ++ static/constant/env.go | 2 + static/constant/string.go | 5 + static/env/env.go | 4 + utils/db/dump.go | 79 +++++++ utils/db/dump_test.go | 15 ++ utils/db/mysql/dump.go | 77 +++++++ utils/db/mysql/mysql.go | 46 ++++ 28 files changed, 334 insertions(+), 872 deletions(-) delete mode 100644 main.go delete mode 100644 modules/pusher/README_ZH.md delete mode 100644 modules/pusher/component/oss/kodo/kodo.go delete mode 100644 modules/pusher/component/oss/kodo/request.go delete mode 100644 modules/pusher/component/oss/oss.go delete mode 100644 modules/pusher/config/example.toml delete mode 100644 modules/pusher/main.go delete mode 100644 modules/pusher/modes/client.go delete mode 100644 modules/pusher/modes/config.go delete mode 100644 modules/pusher/modes/pusher/pusher.go delete mode 100644 modules/pusher/modes/syncer/syncer.go delete mode 100644 modules/pusher/modes/task.go delete mode 100644 modules/pusher/share/errors.go delete mode 100644 modules/pusher/share/interface.go delete mode 100644 modules/pusher/share/share.go delete mode 100644 modules/pusher/utils/utils.go delete mode 100644 modules/pusher/utils/watcher.go create mode 100644 static/constant/database.go create mode 100644 static/constant/string.go create mode 100644 utils/db/dump.go create mode 100644 utils/db/dump_test.go create mode 100644 utils/db/mysql/dump.go create mode 100644 utils/db/mysql/mysql.go diff --git a/README.md b/README.md index 983af08..932dcb3 100644 --- a/README.md +++ b/README.md @@ -14,10 +14,10 @@ go-tools contains several tools ## Usage +- [DB backup](utils/db/dump.go) - [DNS](modules/dns/README.md) - [pusher](./pusher/README.md) - [sgs delay](./sgs/README.md) -- [copier](modules/copier/README.md) - algorithm ## Copier diff --git a/go.mod b/go.mod index c8cf6cf..369c977 100644 --- a/go.mod +++ b/go.mod @@ -5,41 +5,47 @@ go 1.21 require ( github.com/emirpasic/gods v1.18.1 github.com/google/martian v2.1.0+incompatible - github.com/pkg/errors v0.9.1 - github.com/qiniu/go-sdk/v7 v7.17.1 + github.com/magiconair/properties v1.8.7 github.com/spf13/cast v1.5.1 - github.com/spf13/cobra v1.7.0 - github.com/spf13/pflag v1.0.5 github.com/spf13/viper v1.16.0 github.com/stretchr/testify v1.8.4 github.com/tealeg/xlsx/v3 v3.3.0 - golang.org/x/net v0.17.0 + github.com/tickstep/aliyunpan-api v0.2.1 + gorm.io/driver/mysql v1.5.6 + gorm.io/gorm v1.25.10 ) require ( github.com/davecgh/go-spew v1.1.1 // indirect github.com/frankban/quicktest v1.14.5 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect + github.com/go-sql-driver/mysql v1.7.0 // indirect github.com/google/btree v1.0.0 // indirect github.com/google/go-cmp v0.5.9 // indirect github.com/hashicorp/hcl v1.0.0 // indirect - github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/jinzhu/inflection v1.0.0 // indirect + github.com/jinzhu/now v1.1.5 // indirect + github.com/json-iterator/go v1.1.12 // indirect github.com/kr/pretty v0.3.1 // indirect github.com/kr/text v0.2.0 // indirect - github.com/magiconair/properties v1.8.7 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect github.com/pelletier/go-toml/v2 v2.0.8 // indirect github.com/peterbourgon/diskv/v3 v3.0.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/rogpeppe/fastuuid v1.2.0 // indirect github.com/rogpeppe/go-internal v1.9.0 // indirect + github.com/satori/go.uuid v1.2.0 // indirect github.com/shabbyrobe/xmlwriter v0.0.0-20200208144257-9fca06d00ffa // indirect github.com/spf13/afero v1.9.5 // indirect github.com/spf13/jwalterweatherman v1.1.0 // indirect + github.com/spf13/pflag v1.0.5 // indirect github.com/subosito/gotenv v1.4.2 // indirect - golang.org/x/sync v0.1.0 // indirect + github.com/tickstep/library-go v0.1.0 // indirect golang.org/x/sys v0.13.0 // indirect golang.org/x/text v0.13.0 // indirect + gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 19708fa..4be793c 100644 --- a/go.sum +++ b/go.sum @@ -38,6 +38,20 @@ cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3f dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= +github.com/btcsuite/btcd v0.20.1-beta/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ= +github.com/btcsuite/btcd v0.22.1/go.mod h1:wqgTSL29+50LRkmOVknEdmt8ZojIzhuWvgu/iptuN7Y= +github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1/go.mod h1:7SFka0XMvUgj3hfZtydOrQY2mwhPclbT2snogU7SQQc= +github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA= +github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg= +github.com/btcsuite/btcutil v1.0.3-0.20201208143702-a53e38424cce/go.mod h1:0DVlHczLPewLcPGEIeUEzfOJhqGPQ0mJJRDBtD307+o= +github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd/go.mod h1:HHNXQzUsZCxOoE+CPiyCTO6x34Zs86zZUiwtpXoGdtg= +github.com/btcsuite/goleveldb v0.0.0-20160330041536-7834afc9e8cd/go.mod h1:F+uVaaLLH7j4eDXPRvw78tMflu7Ie2bzYOH4Y8rRKBY= +github.com/btcsuite/goleveldb v1.0.0/go.mod h1:QiK9vBlgftBg6rWQIj6wFzbPfRjiykIEhBH4obrXJ/I= +github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc= +github.com/btcsuite/snappy-go v1.0.0/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc= +github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY= +github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= @@ -46,11 +60,13 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= -github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/decred/dcrd/lru v1.0.0/go.mod h1:mxKOwFd7lFjN2GZYsiz/ecgqR6kkYAl+0pz0tEMk218= +github.com/denisbrodbeck/machineid v1.0.1/go.mod h1:dJUwb7PTidGDeYyUBmXZ2GphQBbjJCrnectwCyxcUSI= github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= @@ -61,17 +77,14 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.m github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/frankban/quicktest v1.14.5 h1:dfYrrRyLtiqT9GyKXgdh+k4inNeTvmGbuSgZ3lx3GhA= github.com/frankban/quicktest v1.14.5/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= -github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= -github.com/go-playground/locales v0.14.0/go.mod h1:sawfccIbzZTqEDETgFXqTho0QybSa7l++s0DH+LDiLs= -github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= -github.com/go-playground/universal-translator v0.18.0/go.mod h1:UvRDBj+xPUEGrFYl+lu/H90nyDXpg0fqeB/AQUGNTVA= -github.com/go-playground/validator/v10 v10.8.0/go.mod h1:9JhgTzTaE31GZDpH/HSvHiRJrJ3iKAgqqH0Bl/Ocjdk= +github.com/go-sql-driver/mysql v1.7.0 h1:ueSltNNllEqE3qcWBTD0iQd3IpL/6U+mJxLkazJ7YPc= +github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -111,6 +124,7 @@ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= @@ -134,34 +148,52 @@ github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= -github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= +github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= +github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= +github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= +github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= +github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= +github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ= +github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY= github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= +github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/gomega v1.4.1/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= +github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/pelletier/go-toml/v2 v2.0.8 h1:0ctb6s9mE31h0/lhu+J6OPmVeDxJn+kYnJc2jZR9tGQ= github.com/pelletier/go-toml/v2 v2.0.8/go.mod h1:vuYfssBdrU2XDZ9bYydBu6t+6a6PYNcZljzZR9VXg+4= github.com/peterbourgon/diskv/v3 v3.0.1 h1:x06SQA46+PKIUftmEujdwSEpIx8kR+M9eLYsUxeYveU= github.com/peterbourgon/diskv/v3 v3.0.1/go.mod h1:kJ5Ny7vLdARGU3WUuy6uzO6T0nb/2gWcT1JiBvRmb5o= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= -github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/profile v1.5.0 h1:042Buzk+NhDI+DeSAA62RwJL8VAuZUMQZUjCsRz1Mug= github.com/pkg/profile v1.5.0/go.mod h1:qBsxPvzyUincmltOk6iyRVxHYg4adc0OFOv72ZdLa18= @@ -169,26 +201,19 @@ github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qR github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/qiniu/dyn v1.3.0/go.mod h1:E8oERcm8TtwJiZvkQPbcAh0RL8jO1G0VXJMW3FAWdkk= -github.com/qiniu/go-sdk/v7 v7.17.1 h1:UoQv7fBKtzAiD1qZPIvTy62Se48YLKxcCYP9nAwWMa0= -github.com/qiniu/go-sdk/v7 v7.17.1/go.mod h1:nqoYCNo53ZlGA521RvRethvxUDvXKt4gtYXOwye868w= -github.com/qiniu/x v1.10.5/go.mod h1:03Ni9tj+N2h2aKnAz+6N0Xfl8FwMEDRC2PAlxekASDs= github.com/rogpeppe/fastuuid v1.2.0 h1:Ppwyp6VYCF1nvBTXL3trRso7mXMlRrw9ooo375wvi2s= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= -github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= -github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= +github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/shabbyrobe/xmlwriter v0.0.0-20200208144257-9fca06d00ffa h1:2cO3RojjYl3hVTbEvJVqrMaFmORhL6O06qdW42toftk= github.com/shabbyrobe/xmlwriter v0.0.0-20200208144257-9fca06d00ffa/go.mod h1:Yjr3bdWaVWyME1kha7X0jsz3k2DgXNa1Pj3XGyUAbx8= github.com/spf13/afero v1.9.5 h1:stMpOSZFs//0Lv29HduCmli3GUfpFoF3Y1Q/aXj/wVM= github.com/spf13/afero v1.9.5/go.mod h1:UBogFpq8E9Hx+xc5CNTTEpTnuHVmXDwZcZcE1eb/UhQ= github.com/spf13/cast v1.5.1 h1:R+kOtfhWQE6TVQzY+4D7wJLBgkdVasCEFxSUBYBYIlA= github.com/spf13/cast v1.5.1/go.mod h1:b9PdjNptOpzXr7Rq1q9gJML/2cdGQAo69NKzQ10KN48= -github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= -github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= @@ -199,6 +224,7 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= @@ -212,27 +238,31 @@ github.com/subosito/gotenv v1.4.2 h1:X1TuBLAMDFbaTAChgCBLu3DU3UPyELpnF2jjJ2cz/S8 github.com/subosito/gotenv v1.4.2/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0= github.com/tealeg/xlsx/v3 v3.3.0 h1:GTm5dBwjHIclUGP8nSdxZ4WDAe0op9Y8lVdGnM/81/s= github.com/tealeg/xlsx/v3 v3.3.0/go.mod h1:89pBNWeVVSonnnrL2V2SjIvdel0DU8XDi7W0XsNSzfk= +github.com/tickstep/aliyunpan-api v0.2.1 h1:bPJGnLR2uwWZ5xNInH2xvVRynlLN3fNGk7CZeM7+TbY= +github.com/tickstep/aliyunpan-api v0.2.1/go.mod h1:SN+N5Vh3lLq3dVimlxKxviCl50gLGXZO96nHHc3qd2o= +github.com/tickstep/library-go v0.1.0 h1:lzz6wpYCeliSzmVISa7rp9bGB4r+uIa9/VUTtk87DFU= +github.com/tickstep/library-go v0.1.0/go.mod h1:uAHeNOIpoywCzlaeLrWmmRSupn03m9kJVZKOEmuarmA= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= +golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200115085410-6d4e4cb37c7d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= -golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -266,9 +296,10 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/net v0.0.0-20180719180050-a680a1efc54d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -299,10 +330,6 @@ golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= -golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= -golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -322,10 +349,8 @@ golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= -golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -360,16 +385,10 @@ golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -378,7 +397,6 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -431,7 +449,6 @@ golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= -golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -529,13 +546,20 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gorm.io/driver/mysql v1.5.6 h1:Ld4mkIickM+EliaQZQx3uOJDJHtrd70MxAUqWqlx3Y8= +gorm.io/driver/mysql v1.5.6/go.mod h1:sEtPWMiqiN1N1cMXoXmBbd8C6/l+TESwriotuRRpkDM= +gorm.io/gorm v1.25.7/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8= +gorm.io/gorm v1.25.10 h1:dQpO+33KalOA+aFYGlK+EfxcI5MbO7EP2yYygwh9h+s= +gorm.io/gorm v1.25.10/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/main.go b/main.go deleted file mode 100644 index e8a724f..0000000 --- a/main.go +++ /dev/null @@ -1,7 +0,0 @@ -package main - -import "github.com/alomerry/go-tools/cmd" - -func main() { - cmd.Execute() -} diff --git a/modules/pusher/README_ZH.md b/modules/pusher/README_ZH.md deleted file mode 100644 index 910f3c5..0000000 --- a/modules/pusher/README_ZH.md +++ /dev/null @@ -1,60 +0,0 @@ -# go-pusher - -go-pusher 是一个可以帮助您上传文件到 OSS 并定期将其备份到 VPS 上的工具 - -## 功能 - -- [x] 上传指定目录下 OSS 中不存在或和 OSS 中不一致的文件 -- [ ] 定时备份本地文件到 VPS 上 - -## 使用 - -- 构建二进制文件 - - `go build main.go` - -- 添加可执行权限 - - `chmod +x ./main` - -- 通过配置文件运行 - - `./main -configPath "配置文件绝对路径"` - -## 配置文件 - -以下是配置的内容： - -```toml -modes = ["pusher", "syncer"] - -[syncer] -# 本地绝对路径 -local-path = "xxx" -# 绝对路径 -remote-path = "xxx" -# 检查文件变动间隔（秒） -interval = 1 - -[pusher] -# oss -oss-provider = "qiniu" -# 待推送目录对应 oss 的前缀 -oss-object-prefix = "blog/public" -# 推送到 oss 超时时间（秒） -push-timeout = 60 -# 本地待检测文件夹绝对路径 -local-directory = "/path/to/push" -# 是否和本地强一致，即本地不存在的文件会从 oss 中删除 -oss-delete-not-exists = false - -[oss-qiniu] -bucket = "xxx" -# 存储区域 -# 华东 z0、华东浙江 2 区 cn-east-2、华北 z1、华南 z2、北美 na0、新加坡 as0、亚太首尔 1 区 fog-cn-east-1（详见 https://developer.qiniu.com/kodo/1238/go） -region = "ZoneHuadong" -access-key = "xxx" -sercet-key = "xxx" -``` - -## 为什么写了 go-pusher diff --git a/modules/pusher/component/oss/kodo/kodo.go b/modules/pusher/component/oss/kodo/kodo.go deleted file mode 100644 index c14d2c1..0000000 --- a/modules/pusher/component/oss/kodo/kodo.go +++ /dev/null @@ -1,115 +0,0 @@ -package kodo - -import ( - "errors" - "fmt" - - "github.com/qiniu/go-sdk/v7/auth/qbox" - "github.com/qiniu/go-sdk/v7/storage" - "github.com/spf13/cast" - "github.com/spf13/viper" - "golang.org/x/net/context" -) - -const ( - ridHuadong = "z0" // 华东 - ridHuadongZheJiang2 = "cn-east-2" // 华东浙江 2 区 - ridHuabei = "z1" // 华北 - ridHuanan = "z2" // 华南 - ridNorthAmerica = "na0" // 北美 - ridSingapore = "as0" // 新加坡 - ridFogCnEast1 = "fog-cn-east-1" // 亚太首尔 1 区 -) - -var ( - kodo Kodo -) - -type Kodo struct { - accessKey string - secretKey string - bucket string - region string - - client *qbox.Mac - bucketManager *storage.BucketManager -} - -func InitKodo() { - kodo = Kodo{ - accessKey: cast.ToString(viper.GetStringMap("oss-qiniu")["access-key"]), - secretKey: cast.ToString(viper.GetStringMap("oss-qiniu")["sercet-key"]), - bucket: cast.ToString(viper.GetStringMap("oss-qiniu")["bucket"]), - } -} - -func GetKodoClient() Kodo { - return kodo -} - -func (k Kodo) getClient() *qbox.Mac { - if k.client != nil { - return k.client - } - k.client = qbox.NewMac(k.accessKey, k.secretKey) - return k.client -} - -func (k Kodo) getBucketManager() *storage.BucketManager { - if k.bucketManager != nil { - return k.bucketManager - } - k.bucketManager = storage.NewBucketManager(k.getClient(), &storage.Config{}) - return k.bucketManager -} - -func (k Kodo) PutFile(filePath, ossKey string) (string, error) { - putPolicy := storage.PutPolicy{ - Scope: fmt.Sprintf("%s:%s", k.bucket, ossKey), // 覆盖写入 - ReturnBody: `{"key":"$(key)"}`, - } - upToken := putPolicy.UploadToken(k.getClient()) - - region, _ := storage.GetRegionByID(storage.RegionID(k.region)) - cfg := storage.Config{ - Region: ®ion, - } - formUploader := storage.NewFormUploader(&cfg) - ret := PutRet{} - putExtra := storage.PutExtra{ - Params: map[string]string{}, - } - err := formUploader.PutFile(context.Background(), &ret, upToken, ossKey, filePath, &putExtra) - if err != nil { - return "", err - } - return ret.Key, nil -} - -func (k Kodo) DeleteFiles(ossKeys []string) error { - deleteOptions := make([]string, 0, len(ossKeys)) - for _, key := range ossKeys { - deleteOptions = append(deleteOptions, storage.URIDelete(k.bucket, key)) - } - - rets, err := k.getBucketManager().Batch(deleteOptions) - if len(rets) == 0 { - // 处理错误 - if e, ok := err.(*storage.ErrorInfo); ok { - return errors.New(fmt.Sprintf("batch error, code:%v", e.Code)) - } else { - return err - } - } - var errs []error - for _, ret := range rets { - // 200 为成功 - if ret.Code != 200 { - errs = append(errs, errors.New(ret.Data.Error)) - } - } - if len(errs) > 0 { - return errors.Join(errs...) - } - return nil -} diff --git a/modules/pusher/component/oss/kodo/request.go b/modules/pusher/component/oss/kodo/request.go deleted file mode 100644 index 57016d9..0000000 --- a/modules/pusher/component/oss/kodo/request.go +++ /dev/null @@ -1,6 +0,0 @@ -package kodo - -// 自定义返回值结构体 -type PutRet struct { - Key string -} diff --git a/modules/pusher/component/oss/oss.go b/modules/pusher/component/oss/oss.go deleted file mode 100644 index e313a92..0000000 --- a/modules/pusher/component/oss/oss.go +++ /dev/null @@ -1,65 +0,0 @@ -package oss - -import ( - "sync" - - "github.com/alomerry/go-tools/modules/pusher/component/oss/kodo" - "github.com/alomerry/go-tools/modules/pusher/share" - "github.com/spf13/cast" - "github.com/spf13/viper" -) - -var ( - Client OSSClient - initOnce sync.Once -) - -type OSSClient struct { - providers []string -} - -type OSS interface { - Push(filePath, key string) (string, error) - Delete(keys []string) error -} - -func InitOSS() { - initOnce.Do(func() { - Client = OSSClient{ - providers: cast.ToStringSlice(viper.GetStringMap("pusher")["oss-provider"]), - } - for _, provider := range Client.providers { - switch provider { - case share.OSS_PROVIDER_QI_NIU: - kodo.InitKodo() - default: - panic(share.OSSNotSupport) - } - } - }) - -} - -func (o OSSClient) Push(filePath, key string) (string, error) { - for _, provider := range o.providers { - switch provider { - case share.OSS_PROVIDER_QI_NIU: - return kodo.GetKodoClient().PutFile(filePath, key) - default: - panic(share.OSSNotSupport) - } - } - return "", nil -} - -func (o OSSClient) Delete(keys []string) error { - for _, provider := range o.providers { - switch provider { - case share.OSS_PROVIDER_QI_NIU: - return kodo.GetKodoClient().DeleteFiles(keys) - default: - panic(share.OSSNotSupport) - } - } - return nil -} diff --git a/modules/pusher/config/example.toml b/modules/pusher/config/example.toml deleted file mode 100644 index 9f6ad2a..0000000 --- a/modules/pusher/config/example.toml +++ /dev/null @@ -1,29 +0,0 @@ -modes = ["pusher", "syncer"] - -[syncer] -# 本地绝对路径 -local-path = "xxx" -# 绝对路径 -remote-path = "xxx" -# 检查文件变动间隔（秒） -interval = 1 - -[pusher] -# oss -oss-provider = "qiniu" -# 待推送目录对应 oss 的前缀 -oss-object-prefix = "blog/public" -# 推送到 oss 超时时间（秒） -push-timeout = 60 -# 本地待检测文件夹绝对路径 -local-directory = "/path/to/push" -# 是否和本地强一致，即本地不存在的文件会从 oss 中删除 -oss-delete-not-exists = false - -[oss-qiniu] -bucket = "xxx" -# 存储区域 -# 华东 z0、华东浙江 2 区 cn-east-2、华北 z1、华南 z2、北美 na0、新加坡 as0、亚太首尔 1 区 fog-cn-east-1（详见 https://developer.qiniu.com/kodo/1238/go） -region = "ZoneHuadong" -access-key = "xxx" -sercet-key = "xxx" diff --git a/modules/pusher/main.go b/modules/pusher/main.go deleted file mode 100644 index a2fde32..0000000 --- a/modules/pusher/main.go +++ /dev/null @@ -1,25 +0,0 @@ -package main - -import ( - "github.com/alomerry/go-tools/modules/pusher/modes" - "github.com/spf13/pflag" - "github.com/spf13/viper" - "golang.org/x/net/context" -) - -func init() { - pflag.String("configPath", "./core.toml", "please set the config abstract path") - pflag.Parse() - viper.BindPFlag("configPath", pflag.Lookup("configPath")) -} - -func main() { - ctx := context.Background() - - // TODO add config validate before run - err := modes.IClient.Init(ctx, "TODO").Run(ctx) - - if err != nil { - panic(err) - } -} diff --git a/modules/pusher/modes/client.go b/modules/pusher/modes/client.go deleted file mode 100644 index 8b90e58..0000000 --- a/modules/pusher/modes/client.go +++ /dev/null @@ -1,62 +0,0 @@ -package modes - -import ( - "errors" - "golang.org/x/net/context" - "sync" -) - -var ( - IClient = &Client{} -) - -type Client struct { - config *config -} - -func (c *Client) Init(ctx context.Context, configPath string) *Client { - c.config = initConfig(ctx, configPath) - return c -} - -func (c *Client) Run(ctx context.Context) error { - var ( - err error - errPipe = make(chan error, len(c.config.tasks)) - errDone = make(chan struct{}) - wg = sync.WaitGroup{} - ) - - for i := range c.config.tasks { - task := c.config.tasks[i] - wg.Add(1) - go func() { - defer wg.Done() - innerErr := task.Run(ctx) - if innerErr != nil { - errPipe <- innerErr - } - }() - } - - go func() { - // 合并任务错误 - for innerErr := range errPipe { - if innerErr != nil { - err = errors.Join(err, innerErr) - } - } - errDone <- struct{}{} - }() - - wg.Wait() - - close(errPipe) - - // 错误处理完毕后结束 - select { - case <-errDone: - } - - return err -} diff --git a/modules/pusher/modes/config.go b/modules/pusher/modes/config.go deleted file mode 100644 index ca93068..0000000 --- a/modules/pusher/modes/config.go +++ /dev/null @@ -1,51 +0,0 @@ -package modes - -import ( - "fmt" - "strings" - - "github.com/alomerry/go-tools/modules/pusher/component/oss" - "github.com/alomerry/go-tools/modules/pusher/modes/pusher" - "github.com/alomerry/go-tools/modules/pusher/share" - "github.com/spf13/viper" - "golang.org/x/net/context" -) - -type config struct { - modes []string - - tasks []Task -} - -func initConfig(_ context.Context, configPath string) *config { - conf := initConfigFile(configPath) - - conf.modes = viper.GetStringSlice("modes") - for _, mode := range conf.modes { - var task Task - switch mode { - case share.MODE_PUSHER: - oss.InitOSS() - task = &pusher.Pusher{} - case share.MODE_SYNCER: - //task = &syncer.Syncer{} - } - task.InitConfig() - conf.tasks = append(conf.tasks, task) - } - return conf -} - -func initConfigFile(configPath string) *config { - var ( - rawPath = viper.GetString("configPath") - ) - - configPath = fmt.Sprintf("%s/%s", share.ExPath, strings.TrimPrefix(strings.TrimPrefix(rawPath, share.ExPath), "/")) - - viper.SetConfigFile(configPath) - if err := viper.MergeInConfig(); err != nil { - panic(err) - } - return &config{} -} diff --git a/modules/pusher/modes/pusher/pusher.go b/modules/pusher/modes/pusher/pusher.go deleted file mode 100644 index 78f87c0..0000000 --- a/modules/pusher/modes/pusher/pusher.go +++ /dev/null @@ -1,227 +0,0 @@ -package pusher - -import ( - "bufio" - "fmt" - "io" - "io/fs" - "os" - "sort" - "strings" - "sync" - "time" - - "github.com/alomerry/go-tools/modules/pusher/component/oss" - "github.com/alomerry/go-tools/modules/pusher/share" - "github.com/alomerry/go-tools/modules/pusher/utils" - "github.com/spf13/cast" - "github.com/spf13/viper" - "golang.org/x/net/context" -) - -type Pusher struct { - localDirPath string - ossPrefix string // 待推送目录对应 oss 的前缀 - pushTimeout uint32 // 推送超时时间 单位：秒 - hashCacheMapper map[string]string // 上次缓存的文件 md5 - resultHashMapper map[string]string // 本次遍历的文件 md5 - providers []string - - status string - lock *sync.RWMutex -} - -func (p *Pusher) InitConfig() { - p.localDirPath = cast.ToString(viper.GetStringMap("pusher")["local-directory"]) - p.ossPrefix = cast.ToString(viper.GetStringMap("pusher")["oss-object-prefix"]) - p.pushTimeout = cast.ToUint32(viper.GetStringMap("pusher")["push-timeout"]) - - // 获取或创建 .oss_pusher_hash 文件，将文件列表记录到 map 中 TODO 文件数量过多可能会 oom - p.hashCacheMapper = getCacheHashMap(p.localDirPath) - p.resultHashMapper = make(map[string]string) - - p.providers = cast.ToStringSlice(viper.GetStringMap("pusher")["oss-provider"]) - - p.lock = &sync.RWMutex{} - p.status = share.TASK_STATUS_PENDING -} - -func (p *Pusher) Run(ctx context.Context) error { - defer p.done() - - if p.pushTimeout > 0 { - ctx, _ = context.WithTimeout(ctx, time.Duration(p.pushTimeout)*time.Second) - } - - // 遍历配置中文件夹下的文件，根据文件完整的相对路径l对比 .oss_pusher_hash 中的 md5，如果有新增或变动的文件，则推送到 oss - err := fs.WalkDir(os.DirFS(p.localDirPath), ".", p.upsertFile) - if err != nil { - panic(err) - } - - // 遍历完毕后，如果 hashCacheMapper 非空，则需要检查 oss 有没有对应 key 的文件，如果存在，需要删除（TODO） - err = p.tryDelNotExistsFile() - if err != nil { - panic(err) - } - - // 写入最新版的文件路径对应的 md5 映射到 .oss_pusher_hash 中去 - err = p.writeCache(p.localDirPath) - if err != nil { - panic(err) - } - return nil -} - -func (p *Pusher) Done() bool { - var done bool - p.lock.RLock() - done = p.status == share.TASK_STATUS_DONE - p.lock.RUnlock() - return done -} - -func (p *Pusher) done() { - p.setStatus(share.TASK_STATUS_DONE) -} - -func (p *Pusher) process() { - p.setStatus(share.TASK_STATUS_PROCESSING) -} - -func (p *Pusher) setStatus(status string) { - p.lock.Lock() - p.status = status - p.lock.Unlock() -} - -func getCacheHashMap(cachePath string) map[string]string { - hashCacheMapper := make(map[string]string) - cacheFile := fmt.Sprintf("%s/%s", cachePath, share.CACHE_FILE_NAME) - _, err := os.Stat(cacheFile) - if err != nil { - // TODO 不存在则创建 - if os.IsNotExist(err) { - _, err = os.Create(cacheFile) - } - if err != nil { - panic(".oss_pusher_hash not exists or create failed.") - } - } else { - file, err := os.OpenFile(cacheFile, os.O_RDONLY, 0) - if err != nil { - panic(err) - } - defer file.Close() - reader := bufio.NewReader(file) - for { - line, isPrefix, err := reader.ReadLine() - if len(line) > 0 && err != nil { - panic(err) - } - if isPrefix { - panic("file too large") - } - if err != nil { - if err != io.EOF { - panic(err) - } - break - } - infos := strings.Split(string(line), "[@]") - hashCacheMapper[infos[0]] = infos[1] - } - } - return hashCacheMapper -} - -func (p *Pusher) upsertFile(relatePath string, d fs.DirEntry, err error) error { - if err != nil { - panic(err) - } - if d.IsDir() { - return nil - } - if needIgnore(relatePath) { - return nil - } - hash, err := utils.FileMD5(fmt.Sprintf("%s/%s", p.localDirPath, relatePath)) - if err != nil { - panic(err) - } - - localFilePath := fmt.Sprintf("%s/%s", p.localDirPath, relatePath) - ossFilePath := fmt.Sprintf("%s/%s", p.ossPrefix, relatePath) - cacheHash, exist := p.hashCacheMapper[relatePath] - // oss 不存在或文件有变动时 upsert 到 oss - if !exist || cacheHash != hash { - key, err := oss.Client.Push(localFilePath, ossFilePath) - if err != nil { - panic(err) - } - - fmt.Printf("推送文件 [%v] 到 oss [%v:%v]\n", localFilePath, ossFilePath, key) - } - - p.resultHashMapper[relatePath] = hash - delete(p.hashCacheMapper, relatePath) - return nil -} - -func (p *Pusher) tryDelNotExistsFile() error { - needDelNotExists := cast.ToBool(viper.GetStringMap("pusher")["oss-delete-not-exists"]) - var batchDeleteKeys []string - for key := range p.hashCacheMapper { - if needDelNotExists { - ossFilePath := fmt.Sprintf("%s/%s", p.ossPrefix, key) - batchDeleteKeys = append(batchDeleteKeys, ossFilePath) - } - } - - if len(batchDeleteKeys) > 0 { - oss.Client.Delete(batchDeleteKeys) - fmt.Printf("从 oss 中删除文件:\n - %v ", strings.Join(batchDeleteKeys, "\n - ")) - } - return nil -} - -func needIgnore(filepath string) bool { - if filepath == ".oss_pusher_hash" { - return true - } - - if strings.HasSuffix(filepath, ".toml") { - return true - } - - if strings.HasSuffix(filepath, ".DS_Store") { - return true - } - return false -} - -func (p *Pusher) writeCache(dirPath string) error { - resultBuilder := &strings.Builder{} - // 按文件名排序写入，防止文件频繁变更 - keys := make([]string, 0, len(p.resultHashMapper)) - for key := range p.resultHashMapper { - keys = append(keys, key) - } - needDelNotExists := cast.ToBool(viper.GetStringMap("pusher")["oss-delete-not-exists"]) - for key := range p.hashCacheMapper { - if !needDelNotExists { - keys = append(keys, key) - } - } - sort.Strings(keys) - for i := range keys { - resultBuilder.WriteString(fmt.Sprintf("%s[@]%s\n", keys[i], p.resultHashMapper[keys[i]])) - } - - cacheFile := fmt.Sprintf("%s/%s", dirPath, share.CACHE_FILE_NAME) - err := os.WriteFile(cacheFile, []byte(resultBuilder.String()), os.FileMode(0777)) - if err != nil { - return err - } - return nil -} diff --git a/modules/pusher/modes/syncer/syncer.go b/modules/pusher/modes/syncer/syncer.go deleted file mode 100644 index 5f64bcd..0000000 --- a/modules/pusher/modes/syncer/syncer.go +++ /dev/null @@ -1,50 +0,0 @@ -package syncer - -import ( - "time" - - "github.com/alomerry/go-tools/modules/pusher/utils" - "github.com/spf13/cast" - "github.com/spf13/viper" - "golang.org/x/net/context" -) - -type Syncer struct { - watcher *utils.Watcher -} - -type config struct { - localPath string - remotePath string - interval time.Duration -} - -func (s *Syncer) InitConfig() { - conf := &config{ - localPath: cast.ToString(viper.GetStringMap("syncer")["local-path"]), - remotePath: cast.ToString(viper.GetStringMap("syncer")["remote-path"]), - interval: time.Second * time.Duration(cast.ToInt64(viper.GetStringMap("syncer")["interval"])), - } - s.watcher = utils.GenWatcher(conf) -} - -func (s *Syncer) Run(ctx context.Context) error { - // TODO - return nil -} - -func (s *Syncer) Done() bool { - // TODO - return true -} - -func (c *config) GetLocalPath() string { - return c.localPath -} - -func (c *config) GetRemotePath() string { - return c.remotePath -} -func (c *config) GetInterval() time.Duration { - return c.interval -} diff --git a/modules/pusher/modes/task.go b/modules/pusher/modes/task.go deleted file mode 100644 index f0373ef..0000000 --- a/modules/pusher/modes/task.go +++ /dev/null @@ -1,9 +0,0 @@ -package modes - -import "golang.org/x/net/context" - -type Task interface { - InitConfig() - Run(context.Context) error - Done() bool -} diff --git a/modules/pusher/share/errors.go b/modules/pusher/share/errors.go deleted file mode 100644 index be2f1dc..0000000 --- a/modules/pusher/share/errors.go +++ /dev/null @@ -1,7 +0,0 @@ -package share - -import "errors" - -var ( - OSSNotSupport = errors.New("not support other oss yet.") -) diff --git a/modules/pusher/share/interface.go b/modules/pusher/share/interface.go deleted file mode 100644 index b88aa5e..0000000 --- a/modules/pusher/share/interface.go +++ /dev/null @@ -1,9 +0,0 @@ -package share - -import "time" - -type WatcherGetter interface { - GetLocalPath() string - GetRemotePath() string - GetInterval() time.Duration -} diff --git a/modules/pusher/share/share.go b/modules/pusher/share/share.go deleted file mode 100644 index ea8f3ca..0000000 --- a/modules/pusher/share/share.go +++ /dev/null @@ -1,31 +0,0 @@ -package share - -import ( - "os" - "path/filepath" -) - -const ( - CACHE_FILE_NAME = ".oss_pusher_hash" - - MODE_PUSHER = "pusher" - MODE_SYNCER = "syncer" - - TASK_STATUS_PROCESSING = "processing" - TASK_STATUS_DONE = "done" - TASK_STATUS_PENDING = "pending" - - OSS_PROVIDER_QI_NIU = "qiniu" -) - -var ( - ExPath string -) - -func init() { - ex, err := os.Executable() - if err != nil { - panic(err) - } - ExPath = filepath.Dir(ex) -} diff --git a/modules/pusher/utils/utils.go b/modules/pusher/utils/utils.go deleted file mode 100644 index 08c01ac..0000000 --- a/modules/pusher/utils/utils.go +++ /dev/null @@ -1,27 +0,0 @@ -package utils - -import ( - "crypto/md5" - "encoding/hex" - "io" - "os" -) - -func FileMD5(filePath string) (string, error) { - file, err := os.Open(filePath) - if err != nil { - return "", err - } - hash := md5.New() - _, _ = io.Copy(hash, file) - return hex.EncodeToString(hash.Sum(nil)), nil -} - -func IndexOfArray[T string](item T, array []T) int { - for i := range array { - if item == array[i] { - return i - } - } - return -1 -} diff --git a/modules/pusher/utils/watcher.go b/modules/pusher/utils/watcher.go deleted file mode 100644 index 475de5f..0000000 --- a/modules/pusher/utils/watcher.go +++ /dev/null @@ -1,41 +0,0 @@ -package utils - -import ( - "io/fs" - "os" - "time" - - "github.com/alomerry/go-tools/modules/pusher/share" -) - -type Watcher struct { - share.WatcherGetter - ticker *time.Ticker -} - -func GenWatcher(conf share.WatcherGetter) *Watcher { - return &Watcher{conf, time.NewTicker(conf.GetInterval())} -} - -func (w *Watcher) Watch() { - w.ticker.Stop() - w.ticker.Reset(w.GetInterval()) - for { - <-w.ticker.C - go w.Walk() - } -} - -func (w *Watcher) Walk() { - fs.WalkDir(os.DirFS(w.GetLocalPath()), ".", upsertFile) -} - -func upsertFile(relatePath string, d fs.DirEntry, err error) error { - if d.IsDir() { - return nil - } - //if needIgnore(relatePath) { - // return nil - //} - return nil -} diff --git a/static/constant/database.go b/static/constant/database.go new file mode 100644 index 0000000..86e2cd0 --- /dev/null +++ b/static/constant/database.go @@ -0,0 +1,25 @@ +package constant + +var ( + MySQL = newDatabaseType("mysql") +) + +var ( + Umami = newDatabase(MySQL, "umami") + WalineBlog = newDatabase(MySQL, "waline_blog") +) + +type DatabaseType string + +func newDatabaseType(name string) DatabaseType { + return DatabaseType(name) +} + +type Database struct { + Type DatabaseType + Name string +} + +func newDatabase(dbType DatabaseType, name string) Database { + return Database{dbType, name} +} diff --git a/static/constant/env.go b/static/constant/env.go index a3e5e89..ca548ef 100644 --- a/static/constant/env.go +++ b/static/constant/env.go @@ -3,6 +3,8 @@ package constant const ( ENV = "ENV" + DEBUG = "DEBUG" + KUBECONFIG = "KUBECONFIG" DATABASE_SALT = "DATABASE_SALT" diff --git a/static/constant/string.go b/static/constant/string.go new file mode 100644 index 0000000..0d7f01a --- /dev/null +++ b/static/constant/string.go @@ -0,0 +1,5 @@ +package constant + +const ( + EmptyStr = "" +) diff --git a/static/env/env.go b/static/env/env.go index c6b43f9..84447fe 100644 --- a/static/env/env.go +++ b/static/env/env.go @@ -22,6 +22,10 @@ func GetElasticSearchAK() string { return os.Getenv(constant.ELASTICSEARCH_PASSWORD) } +func Debug() bool { + return os.Getenv(constant.DEBUG) == constant.DEBUG +} + func GetEnv() string { return os.Getenv(constant.ENV) } diff --git a/utils/db/dump.go b/utils/db/dump.go new file mode 100644 index 0000000..37b5875 --- /dev/null +++ b/utils/db/dump.go @@ -0,0 +1,79 @@ +package db + +import ( + "github.com/alomerry/go-tools/static/constant" + "github.com/alomerry/go-tools/utils/db/mysql" +) + +const ( + defaultDumpPath = "/tmp/db-dump" +) + +type DumpTool struct { + dbCfg map[constant.DatabaseType]map[string]any + clientMap map[constant.DatabaseType]innerDumpTool + + dumpPath string +} + +type innerDumpTool interface { + Dump(prefix string, param map[string]any, db constant.Database) (string, error) +} + +type GenDumpCmdParamFunc func(*DumpTool) + +func MySQLDumpCmdParam(user, host, port, password string) GenDumpCmdParamFunc { + return func(tool *DumpTool) { + tool.dbCfg[constant.MySQL] = map[string]any{ + "user": user, + "host": host, + "port": port, + "password": password, + } + } +} + +func SetDumpPath(path string) GenDumpCmdParamFunc { + return func(tool *DumpTool) { + tool.dumpPath = path + } +} + +func NewDumpTool(paramFunc ...GenDumpCmdParamFunc) *DumpTool { + tool := &DumpTool{ + dumpPath: defaultDumpPath, + dbCfg: map[constant.DatabaseType]map[string]any{}, + } + + tool.clientMap = map[constant.DatabaseType]innerDumpTool{ + constant.MySQL: &mysql.DumpTool{}, + } + + for i := range paramFunc { + paramFunc[i](tool) + } + return tool +} + +func (d *DumpTool) DumpDbs(dbs ...constant.Database) ([]string, error) { + var result []string + for i := range dbs { + sql, err := d.dumpDatabase(dbs[i]) + if err != nil { + return nil, err + } + result = append(result, sql) + } + return result, nil +} + +func (d *DumpTool) dumpDatabase(db constant.Database) (string, error) { + switch db.Type { + case constant.MySQL: + break + default: + return constant.EmptyStr, nil + } + + return d.clientMap[db.Type].Dump(d.dumpPath, d.dbCfg[db.Type], db) +} diff --git a/utils/db/dump_test.go b/utils/db/dump_test.go new file mode 100644 index 0000000..a452911 --- /dev/null +++ b/utils/db/dump_test.go @@ -0,0 +1,15 @@ +package db + +import ( + "fmt" + "github.com/alomerry/go-tools/static/constant" + "testing" +) + +func TestDumpDatabase(t *testing.T) { + tool := NewDumpTool( + MySQLDumpCmdParam("root", "alomerry.com", "13306", "e0t=ereFqvpibm}91Y:n"), + SetDumpPath("/Users/alomerry/workspace/go-tools/output"), + ) + fmt.Println(tool.DumpDbs(constant.WalineBlog)) +} diff --git a/utils/db/mysql/dump.go b/utils/db/mysql/dump.go new file mode 100644 index 0000000..019ad12 --- /dev/null +++ b/utils/db/mysql/dump.go @@ -0,0 +1,77 @@ +package mysql + +import ( + "bufio" + "fmt" + "github.com/alomerry/go-tools/static/constant" + "io" + "log" + "os" + "os/exec" + "time" +) + +const ( + mysqldump = "mysqldump" +) + +type DumpTool struct{} + +func (d *DumpTool) Dump(prefix string, params map[string]any, db constant.Database) (string, error) { + var ( + date = time.Now().Format(time.DateOnly) + cmd = exec.Command(mysqldump, append(d.genDumpCmdParam(params), db.Name)...) + dumpSqlPath = fmt.Sprintf("%s/%s-%s.sql", prefix, db.Name, date) + ) + + dumpSql, err := os.OpenFile(dumpSqlPath, os.O_RDWR|os.O_CREATE, 0777) + if err != nil { + return constant.EmptyStr, err + } + defer dumpSql.Close() + + stdout, err := cmd.StdoutPipe() + if err != nil { + return constant.EmptyStr, err + } + defer stdout.Close() + if err := cmd.Start(); err != nil { + log.Fatal(err) + } + + var ( + buff = bufio.NewReader(stdout) + block = make([]byte, 512) + offset = int64(0) + ) + + for { + cnt, err := buff.Read(block) + if err != nil && err != io.EOF { + return constant.EmptyStr, err + } + if 0 == cnt { + break + } + dumpSql.WriteAt(block[:cnt], offset) + if err != nil && err != io.EOF { + return constant.EmptyStr, err + } + offset += int64(cnt) + } + if err := cmd.Wait(); err != nil { + return constant.EmptyStr, err + } + return dumpSqlPath, nil +} + +// genDumpCmdParam +// mysqldump -u -h -P -p +func (*DumpTool) genDumpCmdParam(param map[string]any) []string { + return []string{ + fmt.Sprintf("-u%s", param["user"]), + fmt.Sprintf("-h%s", param["host"]), + fmt.Sprintf("-P%s", param["port"]), + fmt.Sprintf("-p%s", param["password"]), + } +} diff --git a/utils/db/mysql/mysql.go b/utils/db/mysql/mysql.go new file mode 100644 index 0000000..e659ab0 --- /dev/null +++ b/utils/db/mysql/mysql.go @@ -0,0 +1,46 @@ +package mysql + +import ( + "github.com/alomerry/go-tools/static/env" + m "gorm.io/driver/mysql" + "gorm.io/gorm" + "sync" + "time" +) + +var ( + mysql *gorm.DB + once = sync.Once{} +) + +func Instance(dsnGetter func() string) { + once.Do(func() { + var ( + dsn = dsnGetter() + err error + ) + + mysql, err = gorm.Open(m.Open(dsn), &gorm.Config{ + // PrepareStmt: false, // https://gorm.io/zh_CN/docs/performance.html#缓存预编译语句 + }) + if err != nil { + panic(err) + } + + mysql = mysql.Set("gorm:table_options", "ENGINE=InnoDB CHARSET=utf8 auto_increment=1") + sqlDB, err := mysql.DB() + if err != nil { + panic(err) + } + sqlDB.SetMaxOpenConns(100) + sqlDB.SetMaxIdleConns(20) + sqlDB.SetConnMaxLifetime(time.Hour) + }) +} + +func Session() *gorm.DB { + if env.Debug() { + return mysql.Debug().Session(&gorm.Session{}) + } + return mysql.Session(&gorm.Session{}) +}