From d03d9cf985568890c7cd4714765aea01ebeaf869 Mon Sep 17 00:00:00 2001 From: John Oliver Date: Wed, 12 Sep 2018 10:56:46 +0100 Subject: [PATCH] Revert "Refactor updateforesttags to use functions (#358)" (#500) This reverts commit 363d56d1a70b7c75228f1d5c3611d36536d57631. --- git-hg/add-branch-without-modules.sh | 67 +++ git-hg/add-branch.sh | 81 ++++ git-hg/diff-without-getsource.sh | 89 ++-- git-hg/diff.sh | 105 ++--- git-hg/import-common.sh | 16 +- git-hg/mercurialToGitWithSubModules.sh | 430 ------------------ ...rialToGit.sh => mirrorMercurialIntoGit.sh} | 8 + git-hg/setup.sh | 24 + git-hg/setup8.sh | 24 + git-hg/setup9.sh | 24 + git-hg/setupAmber.sh | 24 + git-hg/update-without-modules.sh | 58 +++ git-hg/updateforesttags.sh | 151 ++++++ 13 files changed, 519 insertions(+), 582 deletions(-) create mode 100755 git-hg/add-branch-without-modules.sh create mode 100755 git-hg/add-branch.sh delete mode 100755 git-hg/mercurialToGitWithSubModules.sh rename git-hg/{mercurialToGit.sh => mirrorMercurialIntoGit.sh} (91%) create mode 100755 git-hg/setup.sh create mode 100755 git-hg/setup8.sh create mode 100755 git-hg/setup9.sh create mode 100644 git-hg/setupAmber.sh create mode 100755 git-hg/update-without-modules.sh create mode 100755 git-hg/updateforesttags.sh diff --git a/git-hg/add-branch-without-modules.sh b/git-hg/add-branch-without-modules.sh new file mode 100755 index 000000000..2841b4e56 --- /dev/null +++ b/git-hg/add-branch-without-modules.sh @@ -0,0 +1,67 @@ +#!/bin/bash +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +################################################################################ +# add-branch-without-modules +# +# Local setup of the Git clone branches for the OpenJDK10 (updates) mercurial +# repository +# +# Initial repo will be pushed to git@github.com:AdoptOpenJDK/openjdk-jdk10u.git +# +################################################################################ + +set -euo pipefail + +echo "Import common functionality" +# shellcheck disable=SC1091 +source import-common.sh + +echo "Enter hg" +cd hg || exit 1 + +bpath=$1 +branch=$2 + +echo "Create $bpath" +mkdir -p "$bpath" || exit 1 + +echo "git hg clone $bpath (root)" +git hg clone "http://hg.openjdk.java.net/$bpath" "$bpath/root" || exit 1 + +echo "checkout the $branch" +if [ "$branch" != "" ]; then + cd "$bpath/root" || exit 1 + git-hg checkout "$branch" || exit 1 + cd - || exit 1 +fi + +echo "Enter ../combined" +cd ../combined || exit 1 + +echo "checkout the master branch" +git checkout -b master || exit 1 + +echo "Add remote for (root)" +git remote add "imports/$bpath/root" "../hg/$bpath/root" || exit 1 + +echo "Fetch (root)" +git fetch "imports/$bpath/root" || exit 1 + +echo "Merge (root)" +git merge "imports/$bpath/root/master" -m "Initial merge of (root)" || exit 1 + +echo "Push the tags to the master branch" +git push github master --tags || exit 1 diff --git a/git-hg/add-branch.sh b/git-hg/add-branch.sh new file mode 100755 index 000000000..053efd82f --- /dev/null +++ b/git-hg/add-branch.sh @@ -0,0 +1,81 @@ +#!/bin/bash +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -euo pipefail + +echo "Common defs" + +# shellcheck disable=SC1091 +source import-common.sh + +echo "Enter hg" + +cd hg || exit 1 + +bpath=$1 + +echo "Create $bpath" + +mkdir -p "$bpath" || exit 1 + +echo "Clone $bpath (root)" +git hg clone "http://hg.openjdk.java.net/$bpath" "$bpath/root" || exit 1 + +# shellcheck disable=SC2154 +for module in "${modules[@]}" +do + echo "Clone $bpath -> $module" + git hg clone "http://hg.openjdk.java.net/$bpath/$module" "$bpath/$module" || exit 1 +done + +echo "Exit hg" +echo "Enter combined" + +cd ../combined || exit 1 + +echo "Branch $bpath" + +pwd + +git checkout -b master || exit 1 + +echo "Add remote for (root)" + +git remote add "imports/$bpath/root" "../hg/$bpath/root" || exit 1 + +echo "Fetch (root)" + +git fetch "imports/$bpath/root" || exit 1 + +echo "Merge (root)" + +git merge "imports/$bpath/root/master" -m "Initial merge of (root)" || exit 1 + +# shellcheck disable=SC2154 +for module in "${modules[@]}" +do + echo "Add remote for '$module'" + git remote add "imports/$bpath/$module" "../hg/$bpath/$module" || exit 1 + + echo "Fetch '$module'" + git fetch "imports/$bpath/$module" || exit 1 + + echo "Merge '$module'" + git subtree add --prefix="$module" "imports/$bpath/$module/master" -m "Initial merge of '$module'" || exit 1 +done + +echo "Push" + +git push github master --tags || exit 1 diff --git a/git-hg/diff-without-getsource.sh b/git-hg/diff-without-getsource.sh index 6b46cd47b..882b1fdde 100755 --- a/git-hg/diff-without-getsource.sh +++ b/git-hg/diff-without-getsource.sh @@ -13,77 +13,42 @@ # limitations under the License. # -################################################################################ -# diff-without-getsource -# -# For finding the diff between an AdoptOpenJDK Git repo and the OpenJDK Mercurial -# Repo for Java versions >= jdk10 -# -# 1. Clones the AdoptOpenJDK Git repo for a particular version -# 2. Clones the OpenJDK Mercurial repo for that same version -# 3. Runs a diff between the two -# -################################################################################ - set -euo pipefail -function checkArgs() { - if [ $# -lt 2 ]; then - echo Usage: "$0" '[AdoptOpenJDK Git Repo Version] [OpenJDK Mercurial Root Forest] [OpenJDK Mercurial Version]' - echo "" - echo "e.g. ./diff-without-getsource.sh jdk10u jdk-updates jdk10u" - echo "" - exit 1 - fi -} - -checkArgs $@ - git_repo_version=$1 -hg_root_forest=$2 -hg_repo_version=$3 - -function cleanUp() { - rm -rf openjdk-git openjdk-hg -} +hg_root_forest=${2:-${1}} # for backwards compatibility +hg_repo_version=${3:-${hg_root_forest}} # for backwards compatibility -function cloneRepos() { - echo "AdoptOpenJDK Git Repo Version: ${git_repo_version}" - echo "OpenJDK Mercurial Repo Version: ${hg_root_forest}/${hg_repo_version}" +#cleanup +rm -rf openjdk-git openjdk-hg - git clone -b master "https://github.com/AdoptOpenJDK/openjdk-${git_repo_version}.git" openjdk-git || exit 1 - hg clone "http://hg.openjdk.java.net/${hg_root_forest}/${hg_repo_version}" openjdk-hg || exit 1 -} +echo "git repo version: ${git_repo_version}" +echo "hg repo version: ${hg_root_forest}/${hg_repo_version}" -function runDiff() { - diffNum=$(diff -rq openjdk-git openjdk-hg -x '.git' -x '.hg' -x '.hgtags' | wc -l) +git clone -b master "https://github.com/AdoptOpenJDK/openjdk-${git_repo_version}.git" openjdk-git || exit 1 +hg clone "http://hg.openjdk.java.net/${hg_root_forest}/${hg_repo_version}" openjdk-hg || exit 1 - if [ "$diffNum" -gt 0 ]; then - echo "ERROR - THE DIFF HAS DETECTED UNKNOWN FILES" - diff -rq openjdk-git openjdk-hg -x '.git' -x '.hg' -x '.hgtags' | grep 'only in' || exit 1 - exit 1 - fi -} +diffNum=$(diff -rq openjdk-git openjdk-hg -x '.git' -x '.hg' -x '.hgtags' | wc -l) -function checkTags() { +if [ "$diffNum" -gt 0 ]; then + echo "ERROR - THE DIFF HAS DETECTED UNKNOWN FILES" + diff -rq openjdk-git openjdk-hg -x '.git' -x '.hg' -x '.hgtags' | grep 'only in' || exit 1 + exit 1 +fi - cd openjdk-git || exit 1 - gitTag=$(git describe --tags "$(git rev-list --tags --max-count=1)") || exit 1 - cd - || exit 1 +# get latest git tag - cd openjdk-hg || exit 1 - hgTag=$(hg log -r "." --template "{latesttag}\n") || exit 1 - cd - || exit 1 +cd openjdk-git || exit 1 +gitTag=$(git describe --tags "$(git rev-list --tags --max-count=1)") || exit 1 +cd - || exit 1 - if [ "$gitTag" == "$hgTag" ]; then - echo "Tags are in sync" - else - echo "ERROR - THE TAGS ARE NOT IN SYNC" - exit 1 - fi -} +cd openjdk-hg || exit 1 +hgTag=$(hg log -r "." --template "{latesttag}\n") || exit 1 +cd - || exit 1 -cleanUp -cloneRepos -runDiff -checkTags +if [ "$gitTag" == "$hgTag" ]; then + echo "Tags are in sync" +else + echo "ERROR - THE TAGS ARE NOT IN SYNC" + exit 1 +fi diff --git a/git-hg/diff.sh b/git-hg/diff.sh index 34afa8705..af198691c 100755 --- a/git-hg/diff.sh +++ b/git-hg/diff.sh @@ -13,93 +13,46 @@ # limitations under the License. # -################################################################################ -# diff -# -# For finding the diff between an AdoptOpenJDK Git repo and the OpenJDK Mercurial -# Repo for Java versions <= jdk9u -# -# 1. Clones the AdoptOpenJDK Git repo for a particular version -# 2. Clones the OpenJDK Mercurial repo for that same version -# 3. Runs a diff between the two -# -################################################################################ - -set -euo - -function checkArgs() { - if [ $# -lt 2 ]; then - echo Usage: "$0" '[AdoptOpenJDK Git Repo Version] [OpenJDK Mercurial Root Forest Version]' - echo "" - echo "e.g. ./diff.sh jdk8u jdk8u jdk8u or ./diff.sh jdk9u jdk-updates jdk9u" - echo "" - exit 1 - fi -} - -checkArgs $@ +set -euo pipefail git_repo_version=$1 hg_root_forest=${2:-${1}} # for backwards compatibility hg_repo_version=${3:-${hg_root_forest}} # for backwards compatibility -function cleanUp() { - rm -rf openjdk-git openjdk-hg -} - -function cloneRepos() { - echo "AdoptOpenJDK Git Repo Version: ${git_repo_version}" - echo "OpenJDK Mercurial Repo Version: ${hg_root_forest}/${hg_repo_version}" - - git clone -b master "https://github.com/AdoptOpenJDK/openjdk-${git_repo_version}.git" openjdk-git || exit 1 - hg clone "http://hg.openjdk.java.net/${hg_root_forest}/${hg_repo_version}" openjdk-hg || exit 1 -} - -function updateMercurialClone() { - cd openjdk-hg || exit 1 +#cleanup +rm -rf openjdk-git openjdk-hg - chmod u+x get_source.sh - ./get_source.sh +echo "git repo version: ${git_repo_version}" +echo "hg repo version: ${hg_root_forest}/${hg_repo_version}" - cd - || exit 1 -} +git clone -b master "https://github.com/AdoptOpenJDK/openjdk-${git_repo_version}.git" openjdk-git || exit 1 +hg clone "http://hg.openjdk.java.net/${hg_root_forest}/${hg_repo_version}" openjdk-hg || exit 1 -function runDiff() { - diffNum=$(diff -rq openjdk-git openjdk-hg -x '.git' -x '.hg' -x '.hgtags' -x '.hgignore' -x 'get_source.sh' -x 'README.md' | wc -l) +cd openjdk-hg || exit 1 +bash get_source.sh +cd - || exit 1 - if [ "$diffNum" -gt 0 ]; then - echo "ERROR - THE DIFF HAS DETECTED UNKNOWN FILES" - diff -rq openjdk-git openjdk-hg -x '.git' -x '.hg' -x '.hgtags' -x '.hgignore' -x 'get_source.sh' -x 'README.md' | grep 'Only in' || exit 1 - exit 1 - fi -} +diffNum=$(diff -rq openjdk-git openjdk-hg -x '.git' -x '.hg' -x '.hgtags' | wc -l) -# This function only checks the latest tag, a future enhancement could be to -# check all tags -function checkLatestTag() { +if [ "$diffNum" -gt 0 ]; then + echo "ERROR - THE DIFF HAS DETECTED UNKNOWN FILES" + diff -rq openjdk-git openjdk-hg -x '.git' -x '.hg' -x '.hgtags' | grep 'only in' || exit 1 + exit 1 +fi - cd openjdk-git || exit 1 - gitTag=$(git describe --abbrev=0 --tags) || exit 1 - cd - || exit 1 +# get latest git tag - cd openjdk-hg || exit 1 - hgTag=$(hg log -r "." --template "{latesttag}\n") || exit 1 - cd - || exit 1 +cd openjdk-git || exit 1 +gitTag=$(git describe --abbrev=0 --tags) || exit 1 +cd - || exit 1 - if [ "$gitTag" == "$hgTag" ]; then - echo "Latest Tags are in sync" - else - echo "ERROR - Git tag ${gitTag} is not equal to Hg tag ${hgTag}" - exit 1 - fi -} +cd openjdk-hg || exit 1 +hgTag=$(hg log -r "." --template "{latesttag}\n") || exit 1 +cd - || exit 1 -cleanUp -cloneRepos -updateMercurialClone -runDiff -# No longer run the tag checking as we're only pulling in selective tags for -# AdoptOpenJDK. For others using this script (who are pulling in ALL of the -# tags) you may wish to reenable this function and even enhance it to compare -# all tags. -#checkLatestTag +if [ "$gitTag" == "$hgTag" ]; then + echo "Tags are in sync" +else + echo "ERROR - THE TAGS ARE NOT IN SYNC" + exit 1 +fi diff --git a/git-hg/import-common.sh b/git-hg/import-common.sh index fc0db3407..bb05f0611 100755 --- a/git-hg/import-common.sh +++ b/git-hg/import-common.sh @@ -29,10 +29,7 @@ function checkGitVersion() { GIT_MAJOR_VERSION=$(echo "$GIT_VERSION" | cut -d. -f1) GIT_MINOR_VERSION=$(echo "$GIT_VERSION" | cut -d. -f2) [ "$GIT_MAJOR_VERSION" -eq 1 ] && echo I need git version 2.16 or later and you have "$GIT_VERSION" && exit 1 - if [ "$GIT_MAJOR_VERSION" -eq 2 ] && [ "$GIT_MINOR_VERSION" -lt 16 ] ; then - echo I need git version 2.16 or later and you have "$GIT_VERSION" - exit 1 - fi + [ "$GIT_MAJOR_VERSION" -eq 2 ] && [ "$GIT_MINOR_VERSION" -lt 16 ] && echo I need git version 2.16 or later and you have "$GIT_VERSION" && exit 1 } function installGitRemoteHg() { @@ -41,7 +38,7 @@ function installGitRemoteHg() { echo "Getting it from https://raw.githubusercontent.com/felipec/git-remote-hg/master/git-remote-hg" mkdir -p "$WORKSPACE/bin" PATH="$PATH:$WORKSPACE/bin" - wget -O "$WORKSPACE/bin/git-remote-hg" "https://raw.githubusercontent.com/felipec/git-remote-hg/master/git-remote-hg" + wget -O "$WORKSPACE/bin/git-remote-hg https://raw.githubusercontent.com/felipec/git-remote-hg/master/git-remote-hg" chmod ugo+x "$WORKSPACE/bin/git-remote-hg" if ! which git-remote-hg 2>/dev/null; then echo "Still cannot find it, exiting.." @@ -49,12 +46,3 @@ function installGitRemoteHg() { fi fi } - -# Merge master into dev as we build off dev at the AdoptOpenJDK Build farm -# dev contains patches that AdoptOpenJDK has beyond upstream OpenJDK -function performMergeIntoDevFromMaster() { - git checkout dev || git checkout -b dev - git rebase master || exit 1 - git push origin dev || exit 1 -} - diff --git a/git-hg/mercurialToGitWithSubModules.sh b/git-hg/mercurialToGitWithSubModules.sh deleted file mode 100755 index 3f7cf7ff8..000000000 --- a/git-hg/mercurialToGitWithSubModules.sh +++ /dev/null @@ -1,430 +0,0 @@ -#!/bin/bash -# -# =========================================================================== -# (c) Copyright IBM Corp. 2018 All Rights Reserved -# =========================================================================== -# -# This code is free software; you can redistribute it and/or modify it -# under the terms of the GNU General Public License version 2 only, as -# published by the Free Software Foundation. -# -# This code is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License -# version 2 for more details (a copy is included in the LICENSE file that -# accompanied this code). -# -# You should have received a copy of the GNU General Public License version -# 2 along with this work; if not, see . -# -# =========================================================================== - -################################################################################ -# mercurialToGitWithSubModules -# -# Create a Git clone of a OpenJDK mercurial repository for Java 8/9 complete -# with tags and commit history. -# -# There are three repos involved in the overall operation -# -# REPO_LOCATION - workspace/adoptopenjdk-clone/ - copy of upstream github repo where new commits will end up -# MIRROR - workspace/openjdk-clean-mirror - Unmodified clones of openjdk mercurial (basically a local cache) -# REWRITE_WORKSPACE - workspace/openjdk-rewritten-mirror - Workspace where mercurial is manipulated before being written into the upstream -# - workspace/bin - Helper third party programs -# -# The overall technique used is to: -# -# 1. Setup what tags and modules we want to clone -# 2. Check / Download some tooling we need -# 3. Git clone the remote upstream where we eventually want to push REPO_LOCATION -# 4. Clone / Update $MIRROR -# 5. Rewrite git commits from the $MIRROR into $REWRITE_WORKSPACE which takes -# care of things like prefixing the module name for sub forests. -# 6 Merge the changes in $REWRITE_WORKSPACE into $REPO_LOCATION -# 7. Push that merged result back to the remote upstream of REPO_LOCATION -# -# WARN: Please do not make changes to this script without discussing on the -# build channel on the AdoptOpenJDK Slack. -# -# TODO: Due to the interleaving of commits in mercurial for CPU and PSU updates -# e.g. jdk8u151+b12 and jdk152+b16 this script does not gracefully handle -# merging in both. We recommend following a CPU or PSU strategy only. -# -################################################################################ - -set -euxo - -# Set up the workspace to work from -SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -mkdir -p $SCRIPT_DIR/workspace -WORKSPACE=$SCRIPT_DIR/workspace - -# REPO_LOCATION - workspace/adoptopenjdk-clone/ - copy of upstream github repo where new commits will end up -# MIRROR - workspace/openjdk-clean-mirror - Unmodified clones of openjdk mercurial (basically a local cache) -# REWRITE_WORKSPACE - workspace/openjdk-rewritten-mirror - Workspace where mercurial is manipulated before being written into the upstream -# - workspace/bin - Helper third party programs -MIRROR=$WORKSPACE/openjdk-clean-mirror -REWRITE_WORKSPACE=$WORKSPACE/openjdk-rewritten-mirror/ -REPO_LOCATION=$WORKSPACE/adoptopenjdk-clone/ - -# Setup workspace to work in. -function setupWorkingDir() { - if [ -z ${DEBUG_SCRIPT+x} ]; then - rm -rf "$REWRITE_WORKSPACE" - mkdir -p "$REWRITE_WORKSPACE" - else - rm -rf "$REWRITE_WORKSPACE/root" - rm -rf "$REPO_LOCATION" - fi -} - -setupWorkingDir - -echo "Import common functionality" -# shellcheck disable=SC1091 -source import-common.sh - -function checkArgs() { - if [ $# -lt 3 ]; then - echo Usage: "$0" '[jdk8u|jdk9u] [TARGET_PROJECT] [TARGET_REPO] (TAGS)' - echo "" - echo "e.g. ./mercurialToGitWithSubModules.sh jdk8u git@github.com:AdoptOpenJDK openjdk-jdk8u jdk8u172-b11 jdk8u181-b13 HEAD" - echo "" - exit 1 - fi - if [ -z "$WORKSPACE" ] || [ ! -d "$WORKSPACE" ] ; then - echo Cannot access \$WORKSPACE: "$WORKSPACE" - exit 2 - fi -} - -checkArgs $@ - -# These the the modules in the mercurial forest that we'll have to iterate over -MODULES=(corba langtools jaxp jaxws nashorn jdk hotspot) - -# OpenJDK version that we're wanting to mirror, e.g. jdk8u or jdk9u -OPENJDK_VERSION="$1" -shift - -# Get the target project, e.g. git@github.com:AdoptOpenJDK -TARGET_PROJECT="$1" -shift - -# Get the target repo, e.g. "openjdk-jdk8u" -TARGET_REPO="$1" -shift - -# Get a list of optional tags -TAGS="$@" - -function setMercurialRepoAndTagsToRetrieve() { - case "$OPENJDK_VERSION" in - jdk8*) HG_REPO=http://hg.openjdk.java.net/jdk8u/jdk8u - # Would be nice to pull out the tags in an automated fashion, but - # OpenJDK does not provide this yet. HEAD is interpreted by the - # script below to mean tip/latest commit. - # Skipping jdk8u152-b16 as it seems to be problematic - if [ -z "$TAGS" ] ; then - TAGS="jdk8u144-b34 jdk8u162-b12 jdk8u172-b11 jdk8u181-b13 HEAD" - fi;; - jdk9*) HG_REPO=http://hg.openjdk.java.net/jdk-updates/jdk9u - - if [ -z "$TAGS" ] ; then - TAGS="jdk-9+181 jdk-9.0.1+11 jdk-9.0.3+9 jdk-9.0.4+11 HEAD" - fi;; - *) Unknown JDK version - only jdk8u and jdk9u are supported; exit 1;; - esac -} - -# Clone current Git repo -function cloneGitOpenJDKRepo() { - echo "Clone current $TARGET_REPO" - if [ ! -d "$REPO_LOCATION" ] ; then - mkdir -p "$REPO_LOCATION" - cd "$REPO_LOCATION" - git clone "$TARGET_PROJECT/$TARGET_REPO.git" . - cd "$REPO_LOCATION" || exit 1; - else - cd "$REPO_LOCATION" - git fetch origin - git reset --hard origin/master - fi - - commitCount=$(git rev-list --all --count) - if [ "$commitCount" == 0 ]; then - git checkout -b master - touch README.md - git add README.md - git commit -a -m "Initial commit" - git push --set-upstream origin master - fi - - git checkout master - git pull --allow-unrelated-histories origin master - git fetch --tags - local oldtag=$(git describe --abbrev=0 --tags) - echo "Current openjdk level is $oldtag" -} - -function updateRepo() { - repoName=$1 - repoLocation=$2 - - if [ ! -d "$MIRROR/$repoName/.git" ]; then - rm -rf "$MIRROR/$repoName" || exit 1 - mkdir -p "$MIRROR/$repoName" || exit 1 - cd "$MIRROR/$repoName" - git clone "hg::${repoLocation}" . - fi - - cd "$MIRROR/$repoName" - git fetch origin - git pull origin - git reset --hard origin/master - git fetch --all - -} - -TMP_WORKSPACE="/tmp/adopt-tmp/" - -cleanup () { - exit_code=$? - if [ -d "$TMP_WORKSPACE" ]; then - rm -rf "$TMP_WORKSPACE" || true - fi - - if [ -d "/dev/shm/adopt/" ]; then - rm -rf "/dev/shm/adopt/" || true - fi - exit $exit_code -} - -function updateMirrors() { - mkdir -p "$MIRROR" - cd "$MIRROR" || exit 1 - - updateRepo "root" "${HG_REPO}" - - for module in "${MODULES[@]}" ; do - updateRepo "$module" "${HG_REPO}/$module" - done - - rewriteMirror -} - -function rewriteMirror() { - # If running locally on Mac OS X then use availableMemory=400 - availableMemory=$(free -mw | grep Mem | egrep -o "[0-9]+$") - - trap cleanup EXIT ERR INT TERM - - if [[ $availableMemory -gt 500 ]]; then - echo "Detected more than 500mb of ram available, attempting to use ram dist to speed up" - TMP_WORKSPACE="/dev/shm/adopt/" - else - TMP_WORKSPACE="/tmp/adopt-tmp/" - rm -rf "$TMP_WORKSPACE" || true - fi - - mkdir -p "$TMP_WORKSPACE" || exit 1 - - for module in "${MODULES[@]}" ; do - needsUpdate=$(doesModuleNeedUpdate "$module") - - # dont rewrite mirror if HEAD in mirror is already merged in - if [ "$needsUpdate" == "true" ]; then - - mkdir -p "$REWRITE_WORKSPACE/$module" - cd "$REWRITE_WORKSPACE/$module" || exit 1 - - # Clone the sub module - echo "$(date +%T)": "Clone $module" - git clone "$MIRROR/$module" . || exit 1 - - # Get to to the tag that we want - git fetch --tags - - echo "$(date +%T)": "GIT filter on $module" - mkdir "$TMP_WORKSPACE/$module" - git reset --hard master - - # This looks a bit odd but trust us, take all files and prepend $module to them - prefix_module_sed="s|$(printf '\t')\\\"*|&$module/|" - git filter-branch -d "$TMP_WORKSPACE/$module" -f --index-filter "git rm -f -q --cached --ignore-unmatch .hgignore .hgtags && git ls-files -s | sed \"$prefix_module_sed\" | GIT_INDEX_FILE=\$GIT_INDEX_FILE.new git update-index --index-info && mv \"\$GIT_INDEX_FILE.new\" \"\$GIT_INDEX_FILE\"" --prune-empty --tag-name-filter cat -- --all - - rm -rf "$TMP_WORKSPACE/$module" || exit 1 - fi - done - - rm -rf "$TMP_WORKSPACE" || exit 1 -} - -function checkoutRoot() { - # If we haven't already mirrored the $OPENJDK_VERSION then git clone - if [ ! -d "$REWRITE_WORKSPACE/root/.git" ] ; then - rm -rf "$REWRITE_WORKSPACE/root" || true - mkdir "$REWRITE_WORKSPACE/root" - cd "$REWRITE_WORKSPACE/root" - git clone "$MIRROR/root" . - git remote set-url origin "file://$MIRROR/root" - fi - - # Move into the $OPENJDK_VERSION and make sure we're on the latest master - cd "$REWRITE_WORKSPACE/root" || exit 1 - git pull - git fetch --all - git reset --hard origin/master - - # Remove certain Mercurial specific files from history - (git filter-branch -f --index-filter 'git rm -r -f -q --cached --ignore-unmatch .hg .hgignore .hgtags get_source.sh' --prune-empty --tag-name-filter cat -- --all) - -} - -function fetchRootTagIntoRepo() { - NEWTAG=$1 - - cd "$REWRITE_WORKSPACE/root" || exit 1 - git reset --hard "$NEWTAG" - - echo "$(date +%T)": "Updating master branch for $NEWTAG" - cd "$REPO_LOCATION" || exit 1 - git branch --unset-upstream || true - git checkout master - git fetch "$REWRITE_WORKSPACE/root" - git merge --allow-unrelated-histories -m "Merge base $NEWTAG" FETCH_HEAD -} - - -function fetchModuleTagIntoRepo() { - NEWTAG=$1 - module=$2 - - needsUpdate=$(doesModuleNeedUpdate "$module") - - if [ "$needsUpdate" == "true" ]; then - cd "$REWRITE_WORKSPACE/$module" - newCommitId=$(git rev-list -n 1 "$NEWTAG") - - cd "$MIRROR/$module" - originalCommitId=$(git rev-list -n 1 "$NEWTAG") - - cd "$REPO_LOCATION" || exit 1 - - if [ "$NEWTAG" == "HEAD" ] - then - git fetch "$REWRITE_WORKSPACE/$module" master - else - git fetch "$REWRITE_WORKSPACE/$module" "refs/tags/$NEWTAG" - fi - - echo "$(date +%T)": GIT merge of "$module" - if ! git merge --allow-unrelated-histories -m "Merge module $module at $NEWTAG. OriginalCommitId: $originalCommitId NewCommitId: $newCommitId" FETCH_HEAD; then - if ! tty; then - echo "Aborting - not running on a real tty therefore cannot allow manual intervention" - exit 10 - else - echo "Please resolve them in another window then press return to continue" - read -r _ - fi - echo Please resolve the conflicts above in "$REPO_LOCATION", and press return to continue - read -r _ - fi - fi -} - -function pushTagToMaster() { - - NEWTAG=$1 - - cd "$REPO_LOCATION" || exit 1 - git reset --hard master - git push origin master - - echo "Pulling in changes to $TARGET_REPO branch" - - # Grab anything that someone else may have pushed to the remote - git fetch origin master - if ! git merge --allow-unrelated-histories -m "Merge $NEWTAG into $TARGET_REPO" FETCH_HEAD; then - echo Conflict resolution needed in "$REPO_LOCATION" - if ! tty; then - echo "Aborting - not running on a real tty therefore cannot allow manual intervention" - exit 10 - else - echo "Please resolve them in another window then press return to continue" - read -r _ - fi - fi - - if [ "$NEWTAG" != "HEAD" ] ; then - echo "Override existing tag on the tag from the server if it is present or push will fail" - git tag -f -a "$NEWTAG" -m "Merge $NEWTAG into master" - fi - - if [ "$NEWTAG" != "HEAD" ] ; then - git push origin :refs/tags/"$NEWTAG" - fi - - if [ "$NEWTAG" == "HEAD" ] ; then - git push origin master - fi - - if [ "$NEWTAG" != "HEAD" ] ; then - git push origin master --tags - fi -} - -function doesModuleNeedUpdate() { - module=$1 - - cd "$MIRROR/$module" - latestCommitInMirror=$(git rev-list -n 1 HEAD) - - cd "$REPO_LOCATION" - - # Merge module $module at $NEWTAG. OriginalCommitId: $originalCommitId NewCommitId: $newCommitId, tr -d is for Mac OS X - mergeCount=$(git log --all --pretty=format:"%H\n" --grep="Merge module $module at.*OriginalCommitId: $latestCommitInMirror" | wc -w | tr -d " ") - - if [ "$mergeCount" == 0 ]; then - echo "true" - else - echo "false" - fi -} - -function cloneMercurialOpenJDKRepo() { - - if [ -z ${DEBUG_SCRIPT+x} ]; then - updateMirrors - fi - - cd "$REWRITE_WORKSPACE" || exit 1 - checkoutRoot - - for NEWTAG in $TAGS ; do - - cd "$REPO_LOCATION" || exit 1 - - if git tag | grep "^$NEWTAG$" ; then - echo "Skipping $NEWTAG as it already exists" - else - fetchRootTagIntoRepo "$NEWTAG" - - # For each module - for module in "${MODULES[@]}" ; do - fetchModuleTagIntoRepo "$NEWTAG" "$module" - done - pushTagToMaster "$NEWTAG" - fi - done - - - -} - -setMercurialRepoAndTagsToRetrieve -checkGitVersion -installGitRemoteHg -cloneGitOpenJDKRepo -cloneMercurialOpenJDKRepo -performMergeIntoDevFromMaster diff --git a/git-hg/mercurialToGit.sh b/git-hg/mirrorMercurialIntoGit.sh similarity index 91% rename from git-hg/mercurialToGit.sh rename to git-hg/mirrorMercurialIntoGit.sh index c0e060faf..91220990c 100755 --- a/git-hg/mercurialToGit.sh +++ b/git-hg/mirrorMercurialIntoGit.sh @@ -83,6 +83,14 @@ function performMergeFromMercurialIntoGit() { git push origin "$BRANCH" --tags || exit 1 } +# Merge master into dev as we build off dev at the AdoptOpenJDK Build farm +# dev contains patches that AdoptOpenJDK has beyond upstream OpenJDK +function performMergeIntoDevFromMaster() { + git checkout dev || git checkout -b dev + git rebase master || exit 1 + git push origin dev || exit 1 +} + # TODO Need to cover always merging mercurial master into our GitHub dev branch checkArgs $# #checkGitVersion diff --git a/git-hg/setup.sh b/git-hg/setup.sh new file mode 100755 index 000000000..c133aa725 --- /dev/null +++ b/git-hg/setup.sh @@ -0,0 +1,24 @@ +#!/bin/bash +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +set -euo pipefail +rm -rf "$WORKSPACE/combined" "$WORKSPACE/hg" +mkdir "$WORKSPACE/combined" +mkdir "$WORKSPACE/hg" +cd "$WORKSPACE/combined" || exit 1 +git init +git checkout -b root-commit || exit 1 +git remote add github git@github.com:AdoptOpenJDK/openjdk-"${1}".git +cd - || exit 1 +bash add-branch-without-modules.sh "${2:-jdk/jdk}" diff --git a/git-hg/setup8.sh b/git-hg/setup8.sh new file mode 100755 index 000000000..7d5f08e80 --- /dev/null +++ b/git-hg/setup8.sh @@ -0,0 +1,24 @@ +#!/bin/bash +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +set -euo pipefail +rm -rf "$WORKSPACE/combined" "$WORKSPACE/hg" +mkdir "$WORKSPACE/combined" +mkdir "$WORKSPACE/hg" +cd "$WORKSPACE/combined" || exit 1 +git init +git checkout -b root-commit || exit 1 +git remote add github git@github.com:AdoptOpenJDK/openjdk-jdk8u.git || exit 1 +cd - || exit 1 +bash add-branch.sh jdk8u/jdk8u diff --git a/git-hg/setup9.sh b/git-hg/setup9.sh new file mode 100755 index 000000000..068a7e96e --- /dev/null +++ b/git-hg/setup9.sh @@ -0,0 +1,24 @@ +#!/bin/bash +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +set -euo pipefail +rm -rf "$WORKSPACE/combined" "$WORKSPACE/hg" +mkdir "$WORKSPACE/combined" +mkdir "$WORKSPACE/hg" +cd "$WORKSPACE/combined" || exit 1 +git init +git checkout -b root-commit || exit 1 +git remote add github git@github.com:AdoptOpenJDK/openjdk-jdk9.git +cd - || exit 1 +bash add-branch.sh jdk9/jdk9 diff --git a/git-hg/setupAmber.sh b/git-hg/setupAmber.sh new file mode 100644 index 000000000..50207133b --- /dev/null +++ b/git-hg/setupAmber.sh @@ -0,0 +1,24 @@ +#!/bin/bash +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +set -euo pipefail +rm -rf "$WORKSPACE/combined" "$WORKSPACE/hg" +mkdir "$WORKSPACE/combined" +mkdir "$WORKSPACE/hg" +cd "$WORKSPACE/combined" || exit 1 +git init +git checkout -b root-commit || exit 1 +git remote add github git@github.com:AdoptOpenJDK/openjdk-amber.git +cd - || exit 1 +bash add-branch-without-modules.sh amber/amber raw-string-literal diff --git a/git-hg/update-without-modules.sh b/git-hg/update-without-modules.sh new file mode 100755 index 000000000..d335540e1 --- /dev/null +++ b/git-hg/update-without-modules.sh @@ -0,0 +1,58 @@ +#!/bin/bash +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +echo "Import common functionality" +# shellcheck disable=SC1091 +source import-common.sh + +echo "Enter hg" +cd hg || exit 1 + +# shellcheck disable=SC2035 +bpaths=${1:-$(ls -d -1 */*)} # maintain backward compatibility + +for bpath in $bpaths +do + bpathAsArray=(${bpath/\// }) # for e.g. jdk10/jdk10 or jdk/jdk or openjfx/jfx-dev/rt, becomes: + hg_root_forest=${bpathAsArray[0]} # jdk10 or jdk or openjfx + hg_repo=${bpathAsArray[1]} # jdk10 or jdk or jfx-dev/rt + + pushd "$hg_root_forest/$hg_repo/root" + echo "Update $hg_root_forest/$hg_repo -> (root)" + git hg fetch "http://hg.openjdk.java.net/$hg_root_forest/$hg_repo" + git hg pull "http://hg.openjdk.java.net/$hg_root_forest/$hg_repo" + popd + + echo "Exit hg" + + echo "Enter ../combined" + cd ../combined || exit 1 + + echo "Check out master" + git checkout master || exit 1 + + echo "Fetch (root) of the mercurial forest/repo" + git fetch "imports/$hg_root_forest/$hg_repo/root" || exit 1 + + echo "Merge (root)" + git merge "imports/$hg_root_forest/$hg_repo/root/master" -m "Merge from (root)" --no-ff || exit 1 + + echo "Push to GitHub" + git push github master --tags + + echo "Exit back to ../hg" + cd ../hg || exit 1 +done diff --git a/git-hg/updateforesttags.sh b/git-hg/updateforesttags.sh new file mode 100755 index 000000000..1e4daded4 --- /dev/null +++ b/git-hg/updateforesttags.sh @@ -0,0 +1,151 @@ +#!/bin/bash +# +# =========================================================================== +# (c) Copyright IBM Corp. 2018 All Rights Reserved +# =========================================================================== +# +# This code is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License version 2 only, as +# published by the Free Software Foundation. +# +# This code is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +# version 2 for more details (a copy is included in the LICENSE file that +# accompanied this code). +# +# You should have received a copy of the GNU General Public License version +# 2 along with this work; if not, see . +# +# =========================================================================== + +if [ $# -lt 1 ]; then + echo Usage: "$0" '[jdk8u|jdk9] (TAGS)' + echo "Version supplied should match a repository in AdoptOpenJDK/openjdk-VERSION" + exit 1 +fi +[ -z "$WORKSPACE" ] || [ ! -d "$WORKSPACE" ] && echo Cannot access \$WORKSPACE: "$WORKSPACE" && exit 2 + +MODULES=(corba langtools jaxp jaxws nashorn jdk hotspot) + +OPENJDK_VERSION="$1" +shift +# shellcheck disable=SC2124 +TAGS="$@" +GITHUB_PROJECT=git@github.com:AdoptOpenJDK +GITHUB_REPO="openjdk-$OPENJDK_VERSION" + +case "$OPENJDK_VERSION" in + jdk8*) HG_REPO=http://hg.openjdk.java.net/jdk8u/jdk8u + [ -z "$TAGS" ] && TAGS="jdk8u144-b34 jdk8u151-b12 jdk8u152-b16 jdk8u161-b12 jdk8u162-b12 jdk8u172-b03 jdk8u172-b11";; + jdk9*) HG_REPO=http://hg.openjdk.java.net/jdk-updates/jdk9u + [ -z "$TAGS" ] && TAGS="jdk-9+181 jdk-9.0.1+11 jdk-9.0.3+9 jdk-9.0.4+11";; + *) Unknown JDK version - only jdk8u and jdk9 are supported; exit 1;; +esac + +# Clean up +rm -rf "${WORKSPACE:?}/$GITHUB_REPO" "${WORKSPACE:?}/openjdk" +mkdir -p "$WORKSPACE/$GITHUB_REPO" "$WORKSPACE/openjdk/mirror" + +git --version || exit 1 +GIT_VERSION=$(git --version | awk '{print$NF}') +GIT_MAJOR_VERSION=$(echo "$GIT_VERSION" | cut -d. -f1) +GIT_MINOR_VERSION=$(echo "$GIT_VERSION" | cut -d. -f2) +[ "$GIT_MAJOR_VERSION" -eq 1 ] && echo I need git version 2.16 or later and you have "$GIT_VERSION" && exit 1 +[ "$GIT_MAJOR_VERSION" -eq 2 ] && [ "$GIT_MINOR_VERSION" -lt 16 ] && echo I need git version 2.16 or later and you have "$GIT_VERSION" && exit 1 + +if ! which git-remote-hg 2>/dev/null; then + echo "I need git-remote-hg and could not find it" + echo "Get it from http://raw.githubusercontent.com/felipec/git-remote-hg/master/git-remote-hg" + exit 1 +fi + +# Clone current AdoptOpenJDK repo +cd "$WORKSPACE/$GITHUB_REPO" || exit 1 +echo "Clone current $GITHUB_REPO" +git clone "$GITHUB_PROJECT/$GITHUB_REPO.git" +cd "$GITHUB_REPO" || exit +git fetch --tags +OLDTAG=$(git describe --abbrev=0 --tags) +echo "Current openjdk level is $OLDTAG" + +# Clone current openjdk from Mercurial +echo "Get base openjdk repository" +cd "$WORKSPACE/openjdk/mirror" || exit 1 +git init +git clone --bare "hg::${HG_REPO}" + +cd "$OPENJDK_VERSION.git" || exit 1 + +git filter-branch -f --index-filter 'git rm -r -f -q --cached --ignore-unmatch .hg .hgignore .hgtags get_source.sh' --prune-empty --tag-name-filter cat -- --all + +cd .. || exit +git pull "$OPENJDK_VERSION" +git fetch --tags "$OPENJDK_VERSION" +for NEWTAG in $TAGS ; do + cd "$WORKSPACE/openjdk/mirror" || exit 1 + git reset --hard "$NEWTAG" + + echo "$(date +%T)": "Updating master branch for $NEWTAG" + cd "$WORKSPACE/$GITHUB_REPO/$GITHUB_REPO" || exit 1 + git checkout master + git fetch "$WORKSPACE/openjdk/mirror" + git merge --allow-unrelated-histories -m "Merge base $NEWTAG" FETCH_HEAD + + for module in "${MODULES[@]}" + do + if [ ! -d "$WORKSPACE/openjdk/$module" ]; then + rm -rf "$WORKSPACE/openjdk/$module" + mkdir "$WORKSPACE/openjdk/$module" + cd "$WORKSPACE/openjdk/$module" || exit 1 + git init + echo "$(date +%T)": "Clone $module" + git clone --bare "hg::${HG_REPO}/$module" || exit 1 + echo "$(date +%T)": "GIT filter on $module" + cd "$module.git" || exit + git filter-branch -f --index-filter "git rm -f -q --cached --ignore-unmatch .hgignore .hgtags && git ls-files -s | sed \"s|\t\\\"*|&$module/|\" | GIT_INDEX_FILE=\$GIT_INDEX_FILE.new git update-index --index-info && mv \"\$GIT_INDEX_FILE.new\" \"\$GIT_INDEX_FILE\"" --prune-empty --tag-name-filter cat -- --all + cd .. || exit + fi + echo "$(date +%T)": "GIT pull/reset on $module at $NEWTAG" + cd "$WORKSPACE/openjdk/$module" || exit 1 + git pull "$module" + git fetch --tags "$module" + git reset --hard "$NEWTAG" + cd "$WORKSPACE/$GITHUB_REPO/$GITHUB_REPO" || exit 1 + git fetch "$WORKSPACE/openjdk/$module" + echo "$(date +%T)": GIT filter on "$module" + if ! git merge --allow-unrelated-histories -m "Merge $module at $NEWTAG" FETCH_HEAD; then + if ! tty; then + echo "Aborting - not running on a real tty therefore cannot allow manual intervention" + exit 10 + else + echo "Please resolve them in another window then press return to continue" + read -r _ + fi + echo Please resolve the conflicts above in "$WORKSPACE/$GITHUB_REPO/$GITHUB_REPO", and press return to continue + read -r _ + fi + done + cd "$WORKSPACE/$GITHUB_REPO/$GITHUB_REPO" || exit 1 + git push origin master + + echo "Pulling in changes to $GITHUB_REPO branch" + git checkout master + git fetch origin master + if ! git merge --allow-unrelated-histories -m "Merge $NEWTAG into $GITHUB_REPO" FETCH_HEAD; then + echo Conflict resolution needed in "$WORKSPACE/$GITHUB_REPO/$GITHUB_REPO" + if ! tty; then + echo "Aborting - not running on a real tty therefore cannot allow manual intervention" + exit 10 + else + echo "Please resolve them in another window then press return to continue" + read -r _ + fi + fi + [ "$NEWTAG" != "HEAD" ] && git tag -f -a "$NEWTAG" -m "Merge $NEWTAG into master" + echo Deleting the old version of the tag from the server if it is present or push will fail + # shellcheck disable=SC2015 + [ "$NEWTAG" != "HEAD" ] && git push origin :refs/tags/"$NEWTAG" || true + [ "$NEWTAG" == "HEAD" ] && git push origin master + [ "$NEWTAG" != "HEAD" ] && git push origin master --tags +done