From a71673d5e64dbfb52b77c2a2d4bcd645d8bee658 Mon Sep 17 00:00:00 2001
From: ygqygq2 <ygqygq2@qq.com>
Date: Tue, 18 Jun 2024 08:25:32 +0000
Subject: [PATCH] feat: add scripts

Signed-off-by: ygqygq2 <ygqygq2@qq.com>
---
 .github/ISSUE_TEMPLATE/image-porter.md    |   8 +
 .github/workflows/docker-image-mirror.yml | 148 +++++++++++++
 Jenkinsfile                               |  44 ++++
 README.md                                 |  14 ++
 jenkins_sync_docker_images.sh             | 247 ++++++++++++++++++++++
 5 files changed, 461 insertions(+)
 create mode 100644 .github/ISSUE_TEMPLATE/image-porter.md
 create mode 100644 .github/workflows/docker-image-mirror.yml
 create mode 100644 Jenkinsfile
 create mode 100644 README.md
 create mode 100644 jenkins_sync_docker_images.sh

diff --git a/.github/ISSUE_TEMPLATE/image-porter.md b/.github/ISSUE_TEMPLATE/image-porter.md
new file mode 100644
index 0000000..12efbf5
--- /dev/null
+++ b/.github/ISSUE_TEMPLATE/image-porter.md
@@ -0,0 +1,8 @@
+---
+name: image-porter
+about: docker镜像搬运工
+title: "[PORTER]"
+labels: porter
+assignees: ''
+
+---
diff --git a/.github/workflows/docker-image-mirror.yml b/.github/workflows/docker-image-mirror.yml
new file mode 100644
index 0000000..1b61102
--- /dev/null
+++ b/.github/workflows/docker-image-mirror.yml
@@ -0,0 +1,148 @@
+name: docker image mirror
+
+on:
+  issues:
+    types: [opened]
+
+env:
+  RED: \033[1;31m
+  GREEN: \033[1;32m
+  YELLOW: \033[1;33m
+  BLUE: \033[1;34m
+  PURPLE: \033[1;35m
+  CYAN: \033[1;36m
+  BLANK: \033[0m
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    outputs:
+      DOCKER_IMAGE: ${{ steps.pullIssuesPorter.outputs.DOCKER_IMAGE }}
+      SUCCESS: ${{ steps.successCheck.outputs.SUCCESS }}
+
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+
+    - name: Get porter issues
+      id: pullIssuesPorter
+      uses: actions/github-script@v7
+      with:
+        github-token: ${{secrets.GITHUB_TOKEN}}
+        script: |
+          // 使用 title 获取镜像名和tag
+          const title = context?.payload?.issue?.title;
+          // 使用 body 获取其它参数
+          const body = context?.payload?.issue?.body || '';
+
+          const reg = new RegExp("\\[PORTER\\]", "g");
+          let docker_image = title.replace(reg, "").trim();
+          const issues_author = context?.payload?.issue?.user?.login;
+
+          // 为了防止 image 不带tag，自动添加 latest
+          if(!docker_image.includes(":")) {
+            docker_image = `${docker_image}:latest`
+          }
+
+          let comment_body = '';
+          let is_error = false;
+
+          if( docker_image.includes("@")){
+            is_error = true;
+            comment_body = '@' + issues_author +' 拉取镜像不支持带摘要信息，请去除 @部分'
+          }else{
+            comment_body = `构建进展，详见 [构建任务](https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${{github.run_id}})`
+          }
+
+          const issuesComment = await github.rest.issues.createComment({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              issue_number: context.issue.number,
+              body: comment_body
+          });
+          console.log("create issues comment resp:", issuesComment["status"]);
+
+          if(is_error){
+            core.setFailed("Error");
+          }else if (!docker_image){
+            core.setFailed("No Images");
+          }
+          core.setOutput('DOCKER_IMAGE', docker_image);
+          core.setOutput('BUILD_ARGS', body);
+
+    - name: Retrieve transfer image name
+      id: transferImage
+      run: |
+        echo "${{ steps.pullIssuesPorter.outputs.DOCKER_IMAGE }}" > docker_images.list
+
+    - name: Sync image
+      id: syncImage
+      shell: bash
+      run: |
+        bash jenkins_sync_docker_images.sh docker_images.list
+
+    - name: Success check
+      id: successCheck
+      uses: actions/github-script@v7
+      if: ${{ success() }}
+      with:
+        github-token: ${{secrets.GITHUB_TOKEN}}
+        script: |
+          core.setOutput('SUCCESS', true);
+
+    - name: Close Porter Issues
+      id: closePorterIssues
+      uses: actions/github-script@v7
+      if: ${{ always() }}
+      with:
+        github-token: ${{secrets.GITHUB_TOKEN}}
+        script: |
+          const issuesResponse =  await github.rest.issues.update({
+            owner: context.repo.owner,
+            repo: context.repo.repo,
+            issue_number: context.issue.number,
+            state: 'closed'
+          });
+          console.log("update issues resp:", issuesResponse["status"] == 200 ? "success" : "failed" );
+
+          let comment_body = `转换失败，详见 [构建任务](https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${{github.run_id}})`;
+          let success = String(${{ steps.successCheck.outputs.SUCCESS }}).toLowerCase() == "true";
+          console.log("is success?", success);
+
+          let labels = [];
+          if(success){
+            comment_body = "转换完成，请到你的 Harbor 仓库愉快使用吧 <br/>\n"
+            labels=['success']
+          }else{
+            const jobsResponse = await github.request(`GET /repos/${context.repo.owner}/${context.repo.repo}/actions/runs/${{github.run_id}}/jobs`, {
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              run_id: ${{ github.run_id }}
+            });
+            console.log("jobs",jobsResponse['data']);
+            comment_body += "\n\n 日志:\n\n";
+            for(let job of jobsResponse['data']['jobs']){
+                comment_body += "- [" + job.name + "](" + job.html_url +")";
+            }
+            labels = ['failure'];
+          }
+
+          // 创建 issues comment
+          const issuesComment = await github.rest.issues.createComment({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              issue_number: context.issue.number,
+              body: comment_body
+          });
+          console.log("create issues comment resp:", issuesComment["status"] == 201 ? "success" : "failed" );
+
+          // 更新 issues label
+          if(labels){
+            await github.rest.issues.addLabels({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              issue_number: context.issue.number,
+              labels: labels
+            });
+          }
diff --git a/Jenkinsfile b/Jenkinsfile
new file mode 100644
index 0000000..c1061cd
--- /dev/null
+++ b/Jenkinsfile
@@ -0,0 +1,44 @@
+pipeline {
+  options {
+    // 流水线超时设置
+    timeout(time:1, unit: 'HOURS')
+    //保持构建的最大个数
+    buildDiscarder(logRotator(numToKeepStr: '20'))
+  }
+
+  agent {
+    label "hk-node"
+  }
+
+  environment {
+    // 全局环境变量
+    // SRC_HARBOR_URL = ""  // 源harbor地址
+    // SRC_HARBOR_CRE = credentials('')  // 源harbor用户密码
+    // SRC_HARBOR_REGISTRY = "dev"  // 源harbor项目仓库
+    DEST_HARBOR_URL = "harbor地址"  // 目标harbor地址
+    DEST_HARBOR_CRE = credentials('harbor')  // 目标harbor用户密码
+    // DEST_HARBOR_REGISTRY = "library"  // 目标harbor项目仓库, 已使用input
+  }  
+
+  parameters {
+    // 多行文本输入
+    text(name: 'DOCKER_IMAGES', defaultValue: 'nginx:latest', description: '镜像列表, 一行一个')
+    choice(name: 'DEST_HARBOR_REGISTRY', choices: 'library', description: '选择同步至仓库项目')
+  }
+
+  stages {
+    stage('批量同步docker镜像') {
+      steps {
+        // 写入文件中
+        writeFile file: "docker_images.list", text: "$DOCKER_IMAGES\n", encoding: "UTF-8" 
+          ansiColor('xterm') {
+            echo "#################### 同步镜像开始 ####################"
+            sh """set +x
+              /bin/bash jenkins_sync_docker_images.sh docker_images.list
+            """
+            echo "#################### 同步镜像完成 ####################"
+          }
+      }
+    }
+  }
+}
diff --git a/README.md b/README.md
new file mode 100644
index 0000000..337d898
--- /dev/null
+++ b/README.md
@@ -0,0 +1,14 @@
+# mirror-images
+
+利用“外网”节点同步镜像至私有 Harbor 仓库
+
+## 使用 Jenkins
+
+## 使用 Github action
+请设置 action 环境变量 DEST_HARBOR_URL , secret DEST_HARBOR_CRE_USR 和 DEST_HARBOR_CRE_PSW
+通过新建 Issuse 触发
+
+>标题建议为 `[PORTER]镜像名:tag` 的格式，例如`[PORTER]k8s.gcr.io/pause:3.6`    
+>issues的内容设定为`skopeo copy`的参数，默认为空
+
+其它参数可以参考：[skopeo copy](https://github.com/containers/skopeo/blob/main/docs/skopeo-copy.1.md)
diff --git a/jenkins_sync_docker_images.sh b/jenkins_sync_docker_images.sh
new file mode 100644
index 0000000..e2379f6
--- /dev/null
+++ b/jenkins_sync_docker_images.sh
@@ -0,0 +1,247 @@
+#!/usr/bin/env bash
+
+cd "$(dirname "$0")" || return 1
+SH_DIR=$(pwd)
+ME=$0
+PARAMETERS=$*
+config_file="$1"
+dest_repo="${DEST_HARBOR_URL}/${DEST_HARBOR_REGISTRY}"  # 包含仓库项目的名字
+dest_registry="${DEST_HARBOR_REGISTRY}"
+thread=3 # 此处定义线程数
+faillog="./failure.log" # 此处定义失败列表,注意失败列表会先被删除再重新写入
+echo >> "$config_file"  # 加行空行
+
+# docker login "$SRC_HARBOR_URL" -u "${SRC_HARBOR_CRE_USR}" -p "${SRC_HARBOR_CRE_PSW}"
+docker login "$DEST_HARBOR_URL" -u "${DEST_HARBOR_CRE_USR}" -p "${DEST_HARBOR_CRE_PSW}"
+
+
+#定义输出颜色函数
+function red_echo () {
+#用法:  red_echo "内容"
+    local what="$*"
+    echo -e "$(date +%F-%T) \e[1;31m ${what} \e[0m"
+}
+
+function green_echo () {
+#用法:  green_echo "内容"
+    local what="$*"
+    echo -e "$(date +%F-%T) \e[1;32m ${what} \e[0m"
+}
+
+function yellow_echo () {
+#用法:  yellow_echo "内容"
+    local what="$*"
+    echo -e "$(date +%F-%T) \e[1;33m ${what} \e[0m"
+}
+
+function blue_echo () {
+#用法:  blue_echo "内容"
+    local what="$*"
+    echo -e "$(date +%F-%T) \e[1;34m ${what} \e[0m"
+}
+
+function twinkle_echo () {
+    #用法:  twinkle_echo $(red_echo "内容")  ,此处例子为红色闪烁输出
+    local twinkle='\e[05m'
+    local what="${twinkle} $*"
+    echo -e "$(date +%F-%T) ${what}"
+}
+
+function return_echo () {
+    if [ $? -eq 0 ]; then
+        echo -n "$* " && green_echo "成功"
+        return 0
+    else
+        echo -n "$* " && red_echo "失败"
+        return 1
+    fi
+}
+
+function return_error_exit () {
+    [ $? -eq 0 ] && local REVAL="0"
+    local what=$*
+    if [ "$REVAL" = "0" ];then
+        [ ! -z "$what" ] && { echo -n "$* " && green_echo "成功" ; }
+    else
+        red_echo "$* 失败，脚本退出"
+        exit 1
+    fi
+}
+
+# 定义确认函数
+function user_verify_function () {
+    while true;do
+        echo ""
+        read -p "是否确认?[Y/N]:" Y
+        case $Y in
+            [yY]|[yY][eE][sS])
+                echo -e "answer:  \\033[20G [ \e[1;32m是\e[0m ] \033[0m"
+                break
+            ;;
+            [nN]|[nN][oO])
+                echo -e "answer:  \\033[20G [ \e[1;32m否\e[0m ] \033[0m"
+                exit 1
+            ;;
+            *)
+                continue
+        esac
+    done
+}
+
+# 定义跳过函数
+function user_pass_function () {
+    while true;do
+        echo ""
+        read -p "是否确认?[Y/N]:" Y
+        case $Y in
+            [yY]|[yY][eE][sS])
+                echo -e "answer:  \\033[20G [ \e[1;32m是\e[0m ] \033[0m"
+                break
+                ;;
+            [nN]|[nN][oO])
+                echo -e "answer:  \\033[20G [ \e[1;32m否\e[0m ] \033[0m"
+                return 1
+                ;;
+            *)
+                continue
+        esac
+    done
+}
+
+function check_image() {
+    local image_name=$1
+    local image_tag=$2
+    local encoded
+    encoded=$(perl -MURI::Escape -lne 'chomp; print uri_escape($_)' <<< "$image_name")
+    curl -s -i --connect-timeout 10 -m 20 -u "$DEST_HARBOR_CRE_USR:$DEST_HARBOR_CRE_PSW" -k -X GET \
+        -H "accept: application/json" \
+        "https://$DEST_HARBOR_URL/api/v2.0/projects/$dest_registry/repositories/$encoded/artifacts/$image_tag/tags?page=1&page_size=10&with_signature=false&with_immutable_status=false" \
+        | grep '"name":' > /dev/null
+    return $?
+}
+
+function check_skopeo() {
+    command -v skopeo &> /dev/null
+}
+
+function skopeo_sync_image() {
+    local line=$1
+    local image_name=$2
+    local image_tag=$3
+    skopeo -v && skopeo copy -a \
+        --dest-creds=${DEST_HARBOR_CRE_USR}:${DEST_HARBOR_CRE_PSW} \
+        ${BUILD_ARGS} \
+        docker://${line} \
+        docker://$dest_repo/$image_name:$image_tag
+    return $?
+}
+
+function docker_sync_image() {
+    local line=$1
+    local image_name=$2
+    local image_tag=$3
+    docker pull $line \
+        && docker tag $line $dest_repo/$image_name:$image_tag \
+        && docker push $dest_repo/$image_name:$image_tag \
+        && docker rmi $line \
+        && docker rmi $dest_repo/$image_name:$image_tag \
+    || { red_echo "同步镜像[ $line ]"; echo "$line" | tee -a $faillog ; }
+}
+
+function sync_image() {
+    local line=$*
+    local image_name
+    local image_tag
+    line=$(echo "$line"|sed 's@docker.io/@@g')
+    if [[ ! -z $(echo "$line"|grep '/') ]]; then
+        case $dest_registry in
+            library)
+            image_name=$(echo $line|awk -F':|/' '{print $(NF-2)"/"$(NF-1)}')
+            ;;
+            *)
+            image_name=$(echo $line|awk -F':|/' '{print $(NF-1)}')
+            ;;
+        esac
+        if [[ ! -z $(echo "$image_name"|grep -w "$dest_registry") ]]; then
+            image_name=$(basename $image_name)
+        fi
+    else
+        image_name=$(echo ${line%:*})
+    fi
+    image_tag=$(echo $line|awk -F: '{print $2}')
+    check_image $image_name $image_tag
+    return_echo "检测镜像 [$image_name] 存在 "
+    if [ $? -ne 0 ]; then
+        echo
+        yellow_echo "同步镜像[ $line ]"
+        if [ "$have_skopeo" -eq 0 ]; then
+            skopeo_sync_image "$line" "$image_name" "$image_tag" || docker_sync_image "$line" "$image_name" "$image_tag"
+        else
+            docker_sync_image "$line" "$image_name" "$image_tag"
+        fi
+    else
+        green_echo "已存在镜像，不需要推送[$dest_repo/$image_name:$image_tag]"
+    fi
+}
+
+function usage() {
+    echo "sh $ME config.txt"
+}
+
+if [ -z "$PARAMETERS" ]; then
+    usage
+    exit 55
+fi
+
+function trap_exit() {
+    kill -9 0
+}
+
+function multi_process () {
+    trap 'trap_exit;exit 2' 1 2 3 15
+
+    if [ -f $faillog ];then
+        rm -f $faillog
+    fi
+
+    tmp_fifofile="./$$.fifo"
+    mkfifo $tmp_fifofile      # 新建一个fifo类型的文件
+    exec 6<>$tmp_fifofile      # 将fd6指向fifo类型
+    rm $tmp_fifofile
+
+    for ((i=0;i<$thread;i++)); do
+        echo
+    done >&6 # 事实上就是在fd6中放置了$thread个回车符
+
+    exec 5<$config_file
+    while read line <&5
+    do
+        excute_line=$(echo "$line"|grep -E -v "^#")
+        if [ -z "$excute_line" ]; then
+            continue
+        fi
+        read -u6
+        # 一个read -u6命令执行一次，就从fd6中减去一个回车符，然后向下执行，
+        # fd6中没有回车符的时候，就停在这了，从而实现了线程数量控制
+        { # 此处子进程开始执行，被放到后台
+            sync_image $excute_line
+            echo >&6 # 当进程结束以后，再向fd6中加上一个回车符，即补上了read -u6减去的那个
+        } &
+    done
+
+    wait # 等待所有的后台子进程结束
+    exec 6>&- # 关闭df6
+}
+
+have_skopeo=$(check_skopeo)
+multi_process
+
+if [ -f $faillog ];then
+    red_echo -e "Has failure job"
+    exit 1
+else
+    green_echo "All finish"
+    echo "####################################################"
+fi
+
+exit 0