From 363d56d1a70b7c75228f1d5c3611d36536d57631 Mon Sep 17 00:00:00 2001 From: Martijn Verburg Date: Wed, 12 Sep 2018 10:50:27 +0100 Subject: [PATCH] Refactor updateforesttags to use functions (#358) * Skip 8u152+b16 as it's being a PITA --- git-hg/add-branch-without-modules.sh | 67 --- git-hg/add-branch.sh | 81 ---- git-hg/diff-without-getsource.sh | 89 ++-- git-hg/diff.sh | 105 +++-- git-hg/import-common.sh | 16 +- ...rMercurialIntoGit.sh => mercurialToGit.sh} | 8 - git-hg/mercurialToGitWithSubModules.sh | 430 ++++++++++++++++++ git-hg/setup.sh | 24 - git-hg/setup8.sh | 24 - git-hg/setup9.sh | 24 - git-hg/setupAmber.sh | 24 - git-hg/update-without-modules.sh | 58 --- git-hg/updateforesttags.sh | 151 ------ 13 files changed, 582 insertions(+), 519 deletions(-) delete mode 100755 git-hg/add-branch-without-modules.sh delete mode 100755 git-hg/add-branch.sh rename git-hg/{mirrorMercurialIntoGit.sh => mercurialToGit.sh} (91%) create mode 100755 git-hg/mercurialToGitWithSubModules.sh delete mode 100755 git-hg/setup.sh delete mode 100755 git-hg/setup8.sh delete mode 100755 git-hg/setup9.sh delete mode 100644 git-hg/setupAmber.sh delete mode 100755 git-hg/update-without-modules.sh delete mode 100755 git-hg/updateforesttags.sh diff --git a/git-hg/add-branch-without-modules.sh b/git-hg/add-branch-without-modules.sh deleted file mode 100755 index 2841b4e56..000000000 --- a/git-hg/add-branch-without-modules.sh +++ /dev/null @@ -1,67 +0,0 @@ -#!/bin/bash -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -################################################################################ -# add-branch-without-modules -# -# Local setup of the Git clone branches for the OpenJDK10 (updates) mercurial -# repository -# -# Initial repo will be pushed to git@github.com:AdoptOpenJDK/openjdk-jdk10u.git -# -################################################################################ - -set -euo pipefail - -echo "Import common functionality" -# shellcheck disable=SC1091 -source import-common.sh - -echo "Enter hg" -cd hg || exit 1 - -bpath=$1 -branch=$2 - -echo "Create $bpath" -mkdir -p "$bpath" || exit 1 - -echo "git hg clone $bpath (root)" -git hg clone "http://hg.openjdk.java.net/$bpath" "$bpath/root" || exit 1 - -echo "checkout the $branch" -if [ "$branch" != "" ]; then - cd "$bpath/root" || exit 1 - git-hg checkout "$branch" || exit 1 - cd - || exit 1 -fi - -echo "Enter ../combined" -cd ../combined || exit 1 - -echo "checkout the master branch" -git checkout -b master || exit 1 - -echo "Add remote for (root)" -git remote add "imports/$bpath/root" "../hg/$bpath/root" || exit 1 - -echo "Fetch (root)" -git fetch "imports/$bpath/root" || exit 1 - -echo "Merge (root)" -git merge "imports/$bpath/root/master" -m "Initial merge of (root)" || exit 1 - -echo "Push the tags to the master branch" -git push github master --tags || exit 1 diff --git a/git-hg/add-branch.sh b/git-hg/add-branch.sh deleted file mode 100755 index 053efd82f..000000000 --- a/git-hg/add-branch.sh +++ /dev/null @@ -1,81 +0,0 @@ -#!/bin/bash -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -set -euo pipefail - -echo "Common defs" - -# shellcheck disable=SC1091 -source import-common.sh - -echo "Enter hg" - -cd hg || exit 1 - -bpath=$1 - -echo "Create $bpath" - -mkdir -p "$bpath" || exit 1 - -echo "Clone $bpath (root)" -git hg clone "http://hg.openjdk.java.net/$bpath" "$bpath/root" || exit 1 - -# shellcheck disable=SC2154 -for module in "${modules[@]}" -do - echo "Clone $bpath -> $module" - git hg clone "http://hg.openjdk.java.net/$bpath/$module" "$bpath/$module" || exit 1 -done - -echo "Exit hg" -echo "Enter combined" - -cd ../combined || exit 1 - -echo "Branch $bpath" - -pwd - -git checkout -b master || exit 1 - -echo "Add remote for (root)" - -git remote add "imports/$bpath/root" "../hg/$bpath/root" || exit 1 - -echo "Fetch (root)" - -git fetch "imports/$bpath/root" || exit 1 - -echo "Merge (root)" - -git merge "imports/$bpath/root/master" -m "Initial merge of (root)" || exit 1 - -# shellcheck disable=SC2154 -for module in "${modules[@]}" -do - echo "Add remote for '$module'" - git remote add "imports/$bpath/$module" "../hg/$bpath/$module" || exit 1 - - echo "Fetch '$module'" - git fetch "imports/$bpath/$module" || exit 1 - - echo "Merge '$module'" - git subtree add --prefix="$module" "imports/$bpath/$module/master" -m "Initial merge of '$module'" || exit 1 -done - -echo "Push" - -git push github master --tags || exit 1 diff --git a/git-hg/diff-without-getsource.sh b/git-hg/diff-without-getsource.sh index 882b1fdde..6b46cd47b 100755 --- a/git-hg/diff-without-getsource.sh +++ b/git-hg/diff-without-getsource.sh @@ -13,42 +13,77 @@ # limitations under the License. # +################################################################################ +# diff-without-getsource +# +# For finding the diff between an AdoptOpenJDK Git repo and the OpenJDK Mercurial +# Repo for Java versions >= jdk10 +# +# 1. Clones the AdoptOpenJDK Git repo for a particular version +# 2. Clones the OpenJDK Mercurial repo for that same version +# 3. Runs a diff between the two +# +################################################################################ + set -euo pipefail +function checkArgs() { + if [ $# -lt 2 ]; then + echo Usage: "$0" '[AdoptOpenJDK Git Repo Version] [OpenJDK Mercurial Root Forest] [OpenJDK Mercurial Version]' + echo "" + echo "e.g. ./diff-without-getsource.sh jdk10u jdk-updates jdk10u" + echo "" + exit 1 + fi +} + +checkArgs $@ + git_repo_version=$1 -hg_root_forest=${2:-${1}} # for backwards compatibility -hg_repo_version=${3:-${hg_root_forest}} # for backwards compatibility +hg_root_forest=$2 +hg_repo_version=$3 + +function cleanUp() { + rm -rf openjdk-git openjdk-hg +} -#cleanup -rm -rf openjdk-git openjdk-hg +function cloneRepos() { + echo "AdoptOpenJDK Git Repo Version: ${git_repo_version}" + echo "OpenJDK Mercurial Repo Version: ${hg_root_forest}/${hg_repo_version}" -echo "git repo version: ${git_repo_version}" -echo "hg repo version: ${hg_root_forest}/${hg_repo_version}" + git clone -b master "https://github.com/AdoptOpenJDK/openjdk-${git_repo_version}.git" openjdk-git || exit 1 + hg clone "http://hg.openjdk.java.net/${hg_root_forest}/${hg_repo_version}" openjdk-hg || exit 1 +} -git clone -b master "https://github.com/AdoptOpenJDK/openjdk-${git_repo_version}.git" openjdk-git || exit 1 -hg clone "http://hg.openjdk.java.net/${hg_root_forest}/${hg_repo_version}" openjdk-hg || exit 1 +function runDiff() { + diffNum=$(diff -rq openjdk-git openjdk-hg -x '.git' -x '.hg' -x '.hgtags' | wc -l) -diffNum=$(diff -rq openjdk-git openjdk-hg -x '.git' -x '.hg' -x '.hgtags' | wc -l) + if [ "$diffNum" -gt 0 ]; then + echo "ERROR - THE DIFF HAS DETECTED UNKNOWN FILES" + diff -rq openjdk-git openjdk-hg -x '.git' -x '.hg' -x '.hgtags' | grep 'only in' || exit 1 + exit 1 + fi +} -if [ "$diffNum" -gt 0 ]; then - echo "ERROR - THE DIFF HAS DETECTED UNKNOWN FILES" - diff -rq openjdk-git openjdk-hg -x '.git' -x '.hg' -x '.hgtags' | grep 'only in' || exit 1 - exit 1 -fi +function checkTags() { -# get latest git tag + cd openjdk-git || exit 1 + gitTag=$(git describe --tags "$(git rev-list --tags --max-count=1)") || exit 1 + cd - || exit 1 -cd openjdk-git || exit 1 -gitTag=$(git describe --tags "$(git rev-list --tags --max-count=1)") || exit 1 -cd - || exit 1 + cd openjdk-hg || exit 1 + hgTag=$(hg log -r "." --template "{latesttag}\n") || exit 1 + cd - || exit 1 -cd openjdk-hg || exit 1 -hgTag=$(hg log -r "." --template "{latesttag}\n") || exit 1 -cd - || exit 1 + if [ "$gitTag" == "$hgTag" ]; then + echo "Tags are in sync" + else + echo "ERROR - THE TAGS ARE NOT IN SYNC" + exit 1 + fi +} -if [ "$gitTag" == "$hgTag" ]; then - echo "Tags are in sync" -else - echo "ERROR - THE TAGS ARE NOT IN SYNC" - exit 1 -fi +cleanUp +cloneRepos +runDiff +checkTags diff --git a/git-hg/diff.sh b/git-hg/diff.sh index af198691c..34afa8705 100755 --- a/git-hg/diff.sh +++ b/git-hg/diff.sh @@ -13,46 +13,93 @@ # limitations under the License. # -set -euo pipefail +################################################################################ +# diff +# +# For finding the diff between an AdoptOpenJDK Git repo and the OpenJDK Mercurial +# Repo for Java versions <= jdk9u +# +# 1. Clones the AdoptOpenJDK Git repo for a particular version +# 2. Clones the OpenJDK Mercurial repo for that same version +# 3. Runs a diff between the two +# +################################################################################ + +set -euo + +function checkArgs() { + if [ $# -lt 2 ]; then + echo Usage: "$0" '[AdoptOpenJDK Git Repo Version] [OpenJDK Mercurial Root Forest Version]' + echo "" + echo "e.g. ./diff.sh jdk8u jdk8u jdk8u or ./diff.sh jdk9u jdk-updates jdk9u" + echo "" + exit 1 + fi +} + +checkArgs $@ git_repo_version=$1 hg_root_forest=${2:-${1}} # for backwards compatibility hg_repo_version=${3:-${hg_root_forest}} # for backwards compatibility -#cleanup -rm -rf openjdk-git openjdk-hg +function cleanUp() { + rm -rf openjdk-git openjdk-hg +} + +function cloneRepos() { + echo "AdoptOpenJDK Git Repo Version: ${git_repo_version}" + echo "OpenJDK Mercurial Repo Version: ${hg_root_forest}/${hg_repo_version}" + + git clone -b master "https://github.com/AdoptOpenJDK/openjdk-${git_repo_version}.git" openjdk-git || exit 1 + hg clone "http://hg.openjdk.java.net/${hg_root_forest}/${hg_repo_version}" openjdk-hg || exit 1 +} + +function updateMercurialClone() { + cd openjdk-hg || exit 1 -echo "git repo version: ${git_repo_version}" -echo "hg repo version: ${hg_root_forest}/${hg_repo_version}" + chmod u+x get_source.sh + ./get_source.sh -git clone -b master "https://github.com/AdoptOpenJDK/openjdk-${git_repo_version}.git" openjdk-git || exit 1 -hg clone "http://hg.openjdk.java.net/${hg_root_forest}/${hg_repo_version}" openjdk-hg || exit 1 + cd - || exit 1 +} -cd openjdk-hg || exit 1 -bash get_source.sh -cd - || exit 1 +function runDiff() { + diffNum=$(diff -rq openjdk-git openjdk-hg -x '.git' -x '.hg' -x '.hgtags' -x '.hgignore' -x 'get_source.sh' -x 'README.md' | wc -l) -diffNum=$(diff -rq openjdk-git openjdk-hg -x '.git' -x '.hg' -x '.hgtags' | wc -l) + if [ "$diffNum" -gt 0 ]; then + echo "ERROR - THE DIFF HAS DETECTED UNKNOWN FILES" + diff -rq openjdk-git openjdk-hg -x '.git' -x '.hg' -x '.hgtags' -x '.hgignore' -x 'get_source.sh' -x 'README.md' | grep 'Only in' || exit 1 + exit 1 + fi +} -if [ "$diffNum" -gt 0 ]; then - echo "ERROR - THE DIFF HAS DETECTED UNKNOWN FILES" - diff -rq openjdk-git openjdk-hg -x '.git' -x '.hg' -x '.hgtags' | grep 'only in' || exit 1 - exit 1 -fi +# This function only checks the latest tag, a future enhancement could be to +# check all tags +function checkLatestTag() { -# get latest git tag + cd openjdk-git || exit 1 + gitTag=$(git describe --abbrev=0 --tags) || exit 1 + cd - || exit 1 -cd openjdk-git || exit 1 -gitTag=$(git describe --abbrev=0 --tags) || exit 1 -cd - || exit 1 + cd openjdk-hg || exit 1 + hgTag=$(hg log -r "." --template "{latesttag}\n") || exit 1 + cd - || exit 1 -cd openjdk-hg || exit 1 -hgTag=$(hg log -r "." --template "{latesttag}\n") || exit 1 -cd - || exit 1 + if [ "$gitTag" == "$hgTag" ]; then + echo "Latest Tags are in sync" + else + echo "ERROR - Git tag ${gitTag} is not equal to Hg tag ${hgTag}" + exit 1 + fi +} -if [ "$gitTag" == "$hgTag" ]; then - echo "Tags are in sync" -else - echo "ERROR - THE TAGS ARE NOT IN SYNC" - exit 1 -fi +cleanUp +cloneRepos +updateMercurialClone +runDiff +# No longer run the tag checking as we're only pulling in selective tags for +# AdoptOpenJDK. For others using this script (who are pulling in ALL of the +# tags) you may wish to reenable this function and even enhance it to compare +# all tags. +#checkLatestTag diff --git a/git-hg/import-common.sh b/git-hg/import-common.sh index bb05f0611..fc0db3407 100755 --- a/git-hg/import-common.sh +++ b/git-hg/import-common.sh @@ -29,7 +29,10 @@ function checkGitVersion() { GIT_MAJOR_VERSION=$(echo "$GIT_VERSION" | cut -d. -f1) GIT_MINOR_VERSION=$(echo "$GIT_VERSION" | cut -d. -f2) [ "$GIT_MAJOR_VERSION" -eq 1 ] && echo I need git version 2.16 or later and you have "$GIT_VERSION" && exit 1 - [ "$GIT_MAJOR_VERSION" -eq 2 ] && [ "$GIT_MINOR_VERSION" -lt 16 ] && echo I need git version 2.16 or later and you have "$GIT_VERSION" && exit 1 + if [ "$GIT_MAJOR_VERSION" -eq 2 ] && [ "$GIT_MINOR_VERSION" -lt 16 ] ; then + echo I need git version 2.16 or later and you have "$GIT_VERSION" + exit 1 + fi } function installGitRemoteHg() { @@ -38,7 +41,7 @@ function installGitRemoteHg() { echo "Getting it from https://raw.githubusercontent.com/felipec/git-remote-hg/master/git-remote-hg" mkdir -p "$WORKSPACE/bin" PATH="$PATH:$WORKSPACE/bin" - wget -O "$WORKSPACE/bin/git-remote-hg https://raw.githubusercontent.com/felipec/git-remote-hg/master/git-remote-hg" + wget -O "$WORKSPACE/bin/git-remote-hg" "https://raw.githubusercontent.com/felipec/git-remote-hg/master/git-remote-hg" chmod ugo+x "$WORKSPACE/bin/git-remote-hg" if ! which git-remote-hg 2>/dev/null; then echo "Still cannot find it, exiting.." @@ -46,3 +49,12 @@ function installGitRemoteHg() { fi fi } + +# Merge master into dev as we build off dev at the AdoptOpenJDK Build farm +# dev contains patches that AdoptOpenJDK has beyond upstream OpenJDK +function performMergeIntoDevFromMaster() { + git checkout dev || git checkout -b dev + git rebase master || exit 1 + git push origin dev || exit 1 +} + diff --git a/git-hg/mirrorMercurialIntoGit.sh b/git-hg/mercurialToGit.sh similarity index 91% rename from git-hg/mirrorMercurialIntoGit.sh rename to git-hg/mercurialToGit.sh index 91220990c..c0e060faf 100755 --- a/git-hg/mirrorMercurialIntoGit.sh +++ b/git-hg/mercurialToGit.sh @@ -83,14 +83,6 @@ function performMergeFromMercurialIntoGit() { git push origin "$BRANCH" --tags || exit 1 } -# Merge master into dev as we build off dev at the AdoptOpenJDK Build farm -# dev contains patches that AdoptOpenJDK has beyond upstream OpenJDK -function performMergeIntoDevFromMaster() { - git checkout dev || git checkout -b dev - git rebase master || exit 1 - git push origin dev || exit 1 -} - # TODO Need to cover always merging mercurial master into our GitHub dev branch checkArgs $# #checkGitVersion diff --git a/git-hg/mercurialToGitWithSubModules.sh b/git-hg/mercurialToGitWithSubModules.sh new file mode 100755 index 000000000..3f7cf7ff8 --- /dev/null +++ b/git-hg/mercurialToGitWithSubModules.sh @@ -0,0 +1,430 @@ +#!/bin/bash +# +# =========================================================================== +# (c) Copyright IBM Corp. 2018 All Rights Reserved +# =========================================================================== +# +# This code is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License version 2 only, as +# published by the Free Software Foundation. +# +# This code is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +# version 2 for more details (a copy is included in the LICENSE file that +# accompanied this code). +# +# You should have received a copy of the GNU General Public License version +# 2 along with this work; if not, see . +# +# =========================================================================== + +################################################################################ +# mercurialToGitWithSubModules +# +# Create a Git clone of a OpenJDK mercurial repository for Java 8/9 complete +# with tags and commit history. +# +# There are three repos involved in the overall operation +# +# REPO_LOCATION - workspace/adoptopenjdk-clone/ - copy of upstream github repo where new commits will end up +# MIRROR - workspace/openjdk-clean-mirror - Unmodified clones of openjdk mercurial (basically a local cache) +# REWRITE_WORKSPACE - workspace/openjdk-rewritten-mirror - Workspace where mercurial is manipulated before being written into the upstream +# - workspace/bin - Helper third party programs +# +# The overall technique used is to: +# +# 1. Setup what tags and modules we want to clone +# 2. Check / Download some tooling we need +# 3. Git clone the remote upstream where we eventually want to push REPO_LOCATION +# 4. Clone / Update $MIRROR +# 5. Rewrite git commits from the $MIRROR into $REWRITE_WORKSPACE which takes +# care of things like prefixing the module name for sub forests. +# 6 Merge the changes in $REWRITE_WORKSPACE into $REPO_LOCATION +# 7. Push that merged result back to the remote upstream of REPO_LOCATION +# +# WARN: Please do not make changes to this script without discussing on the +# build channel on the AdoptOpenJDK Slack. +# +# TODO: Due to the interleaving of commits in mercurial for CPU and PSU updates +# e.g. jdk8u151+b12 and jdk152+b16 this script does not gracefully handle +# merging in both. We recommend following a CPU or PSU strategy only. +# +################################################################################ + +set -euxo + +# Set up the workspace to work from +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +mkdir -p $SCRIPT_DIR/workspace +WORKSPACE=$SCRIPT_DIR/workspace + +# REPO_LOCATION - workspace/adoptopenjdk-clone/ - copy of upstream github repo where new commits will end up +# MIRROR - workspace/openjdk-clean-mirror - Unmodified clones of openjdk mercurial (basically a local cache) +# REWRITE_WORKSPACE - workspace/openjdk-rewritten-mirror - Workspace where mercurial is manipulated before being written into the upstream +# - workspace/bin - Helper third party programs +MIRROR=$WORKSPACE/openjdk-clean-mirror +REWRITE_WORKSPACE=$WORKSPACE/openjdk-rewritten-mirror/ +REPO_LOCATION=$WORKSPACE/adoptopenjdk-clone/ + +# Setup workspace to work in. +function setupWorkingDir() { + if [ -z ${DEBUG_SCRIPT+x} ]; then + rm -rf "$REWRITE_WORKSPACE" + mkdir -p "$REWRITE_WORKSPACE" + else + rm -rf "$REWRITE_WORKSPACE/root" + rm -rf "$REPO_LOCATION" + fi +} + +setupWorkingDir + +echo "Import common functionality" +# shellcheck disable=SC1091 +source import-common.sh + +function checkArgs() { + if [ $# -lt 3 ]; then + echo Usage: "$0" '[jdk8u|jdk9u] [TARGET_PROJECT] [TARGET_REPO] (TAGS)' + echo "" + echo "e.g. ./mercurialToGitWithSubModules.sh jdk8u git@github.com:AdoptOpenJDK openjdk-jdk8u jdk8u172-b11 jdk8u181-b13 HEAD" + echo "" + exit 1 + fi + if [ -z "$WORKSPACE" ] || [ ! -d "$WORKSPACE" ] ; then + echo Cannot access \$WORKSPACE: "$WORKSPACE" + exit 2 + fi +} + +checkArgs $@ + +# These the the modules in the mercurial forest that we'll have to iterate over +MODULES=(corba langtools jaxp jaxws nashorn jdk hotspot) + +# OpenJDK version that we're wanting to mirror, e.g. jdk8u or jdk9u +OPENJDK_VERSION="$1" +shift + +# Get the target project, e.g. git@github.com:AdoptOpenJDK +TARGET_PROJECT="$1" +shift + +# Get the target repo, e.g. "openjdk-jdk8u" +TARGET_REPO="$1" +shift + +# Get a list of optional tags +TAGS="$@" + +function setMercurialRepoAndTagsToRetrieve() { + case "$OPENJDK_VERSION" in + jdk8*) HG_REPO=http://hg.openjdk.java.net/jdk8u/jdk8u + # Would be nice to pull out the tags in an automated fashion, but + # OpenJDK does not provide this yet. HEAD is interpreted by the + # script below to mean tip/latest commit. + # Skipping jdk8u152-b16 as it seems to be problematic + if [ -z "$TAGS" ] ; then + TAGS="jdk8u144-b34 jdk8u162-b12 jdk8u172-b11 jdk8u181-b13 HEAD" + fi;; + jdk9*) HG_REPO=http://hg.openjdk.java.net/jdk-updates/jdk9u + + if [ -z "$TAGS" ] ; then + TAGS="jdk-9+181 jdk-9.0.1+11 jdk-9.0.3+9 jdk-9.0.4+11 HEAD" + fi;; + *) Unknown JDK version - only jdk8u and jdk9u are supported; exit 1;; + esac +} + +# Clone current Git repo +function cloneGitOpenJDKRepo() { + echo "Clone current $TARGET_REPO" + if [ ! -d "$REPO_LOCATION" ] ; then + mkdir -p "$REPO_LOCATION" + cd "$REPO_LOCATION" + git clone "$TARGET_PROJECT/$TARGET_REPO.git" . + cd "$REPO_LOCATION" || exit 1; + else + cd "$REPO_LOCATION" + git fetch origin + git reset --hard origin/master + fi + + commitCount=$(git rev-list --all --count) + if [ "$commitCount" == 0 ]; then + git checkout -b master + touch README.md + git add README.md + git commit -a -m "Initial commit" + git push --set-upstream origin master + fi + + git checkout master + git pull --allow-unrelated-histories origin master + git fetch --tags + local oldtag=$(git describe --abbrev=0 --tags) + echo "Current openjdk level is $oldtag" +} + +function updateRepo() { + repoName=$1 + repoLocation=$2 + + if [ ! -d "$MIRROR/$repoName/.git" ]; then + rm -rf "$MIRROR/$repoName" || exit 1 + mkdir -p "$MIRROR/$repoName" || exit 1 + cd "$MIRROR/$repoName" + git clone "hg::${repoLocation}" . + fi + + cd "$MIRROR/$repoName" + git fetch origin + git pull origin + git reset --hard origin/master + git fetch --all + +} + +TMP_WORKSPACE="/tmp/adopt-tmp/" + +cleanup () { + exit_code=$? + if [ -d "$TMP_WORKSPACE" ]; then + rm -rf "$TMP_WORKSPACE" || true + fi + + if [ -d "/dev/shm/adopt/" ]; then + rm -rf "/dev/shm/adopt/" || true + fi + exit $exit_code +} + +function updateMirrors() { + mkdir -p "$MIRROR" + cd "$MIRROR" || exit 1 + + updateRepo "root" "${HG_REPO}" + + for module in "${MODULES[@]}" ; do + updateRepo "$module" "${HG_REPO}/$module" + done + + rewriteMirror +} + +function rewriteMirror() { + # If running locally on Mac OS X then use availableMemory=400 + availableMemory=$(free -mw | grep Mem | egrep -o "[0-9]+$") + + trap cleanup EXIT ERR INT TERM + + if [[ $availableMemory -gt 500 ]]; then + echo "Detected more than 500mb of ram available, attempting to use ram dist to speed up" + TMP_WORKSPACE="/dev/shm/adopt/" + else + TMP_WORKSPACE="/tmp/adopt-tmp/" + rm -rf "$TMP_WORKSPACE" || true + fi + + mkdir -p "$TMP_WORKSPACE" || exit 1 + + for module in "${MODULES[@]}" ; do + needsUpdate=$(doesModuleNeedUpdate "$module") + + # dont rewrite mirror if HEAD in mirror is already merged in + if [ "$needsUpdate" == "true" ]; then + + mkdir -p "$REWRITE_WORKSPACE/$module" + cd "$REWRITE_WORKSPACE/$module" || exit 1 + + # Clone the sub module + echo "$(date +%T)": "Clone $module" + git clone "$MIRROR/$module" . || exit 1 + + # Get to to the tag that we want + git fetch --tags + + echo "$(date +%T)": "GIT filter on $module" + mkdir "$TMP_WORKSPACE/$module" + git reset --hard master + + # This looks a bit odd but trust us, take all files and prepend $module to them + prefix_module_sed="s|$(printf '\t')\\\"*|&$module/|" + git filter-branch -d "$TMP_WORKSPACE/$module" -f --index-filter "git rm -f -q --cached --ignore-unmatch .hgignore .hgtags && git ls-files -s | sed \"$prefix_module_sed\" | GIT_INDEX_FILE=\$GIT_INDEX_FILE.new git update-index --index-info && mv \"\$GIT_INDEX_FILE.new\" \"\$GIT_INDEX_FILE\"" --prune-empty --tag-name-filter cat -- --all + + rm -rf "$TMP_WORKSPACE/$module" || exit 1 + fi + done + + rm -rf "$TMP_WORKSPACE" || exit 1 +} + +function checkoutRoot() { + # If we haven't already mirrored the $OPENJDK_VERSION then git clone + if [ ! -d "$REWRITE_WORKSPACE/root/.git" ] ; then + rm -rf "$REWRITE_WORKSPACE/root" || true + mkdir "$REWRITE_WORKSPACE/root" + cd "$REWRITE_WORKSPACE/root" + git clone "$MIRROR/root" . + git remote set-url origin "file://$MIRROR/root" + fi + + # Move into the $OPENJDK_VERSION and make sure we're on the latest master + cd "$REWRITE_WORKSPACE/root" || exit 1 + git pull + git fetch --all + git reset --hard origin/master + + # Remove certain Mercurial specific files from history + (git filter-branch -f --index-filter 'git rm -r -f -q --cached --ignore-unmatch .hg .hgignore .hgtags get_source.sh' --prune-empty --tag-name-filter cat -- --all) + +} + +function fetchRootTagIntoRepo() { + NEWTAG=$1 + + cd "$REWRITE_WORKSPACE/root" || exit 1 + git reset --hard "$NEWTAG" + + echo "$(date +%T)": "Updating master branch for $NEWTAG" + cd "$REPO_LOCATION" || exit 1 + git branch --unset-upstream || true + git checkout master + git fetch "$REWRITE_WORKSPACE/root" + git merge --allow-unrelated-histories -m "Merge base $NEWTAG" FETCH_HEAD +} + + +function fetchModuleTagIntoRepo() { + NEWTAG=$1 + module=$2 + + needsUpdate=$(doesModuleNeedUpdate "$module") + + if [ "$needsUpdate" == "true" ]; then + cd "$REWRITE_WORKSPACE/$module" + newCommitId=$(git rev-list -n 1 "$NEWTAG") + + cd "$MIRROR/$module" + originalCommitId=$(git rev-list -n 1 "$NEWTAG") + + cd "$REPO_LOCATION" || exit 1 + + if [ "$NEWTAG" == "HEAD" ] + then + git fetch "$REWRITE_WORKSPACE/$module" master + else + git fetch "$REWRITE_WORKSPACE/$module" "refs/tags/$NEWTAG" + fi + + echo "$(date +%T)": GIT merge of "$module" + if ! git merge --allow-unrelated-histories -m "Merge module $module at $NEWTAG. OriginalCommitId: $originalCommitId NewCommitId: $newCommitId" FETCH_HEAD; then + if ! tty; then + echo "Aborting - not running on a real tty therefore cannot allow manual intervention" + exit 10 + else + echo "Please resolve them in another window then press return to continue" + read -r _ + fi + echo Please resolve the conflicts above in "$REPO_LOCATION", and press return to continue + read -r _ + fi + fi +} + +function pushTagToMaster() { + + NEWTAG=$1 + + cd "$REPO_LOCATION" || exit 1 + git reset --hard master + git push origin master + + echo "Pulling in changes to $TARGET_REPO branch" + + # Grab anything that someone else may have pushed to the remote + git fetch origin master + if ! git merge --allow-unrelated-histories -m "Merge $NEWTAG into $TARGET_REPO" FETCH_HEAD; then + echo Conflict resolution needed in "$REPO_LOCATION" + if ! tty; then + echo "Aborting - not running on a real tty therefore cannot allow manual intervention" + exit 10 + else + echo "Please resolve them in another window then press return to continue" + read -r _ + fi + fi + + if [ "$NEWTAG" != "HEAD" ] ; then + echo "Override existing tag on the tag from the server if it is present or push will fail" + git tag -f -a "$NEWTAG" -m "Merge $NEWTAG into master" + fi + + if [ "$NEWTAG" != "HEAD" ] ; then + git push origin :refs/tags/"$NEWTAG" + fi + + if [ "$NEWTAG" == "HEAD" ] ; then + git push origin master + fi + + if [ "$NEWTAG" != "HEAD" ] ; then + git push origin master --tags + fi +} + +function doesModuleNeedUpdate() { + module=$1 + + cd "$MIRROR/$module" + latestCommitInMirror=$(git rev-list -n 1 HEAD) + + cd "$REPO_LOCATION" + + # Merge module $module at $NEWTAG. OriginalCommitId: $originalCommitId NewCommitId: $newCommitId, tr -d is for Mac OS X + mergeCount=$(git log --all --pretty=format:"%H\n" --grep="Merge module $module at.*OriginalCommitId: $latestCommitInMirror" | wc -w | tr -d " ") + + if [ "$mergeCount" == 0 ]; then + echo "true" + else + echo "false" + fi +} + +function cloneMercurialOpenJDKRepo() { + + if [ -z ${DEBUG_SCRIPT+x} ]; then + updateMirrors + fi + + cd "$REWRITE_WORKSPACE" || exit 1 + checkoutRoot + + for NEWTAG in $TAGS ; do + + cd "$REPO_LOCATION" || exit 1 + + if git tag | grep "^$NEWTAG$" ; then + echo "Skipping $NEWTAG as it already exists" + else + fetchRootTagIntoRepo "$NEWTAG" + + # For each module + for module in "${MODULES[@]}" ; do + fetchModuleTagIntoRepo "$NEWTAG" "$module" + done + pushTagToMaster "$NEWTAG" + fi + done + + + +} + +setMercurialRepoAndTagsToRetrieve +checkGitVersion +installGitRemoteHg +cloneGitOpenJDKRepo +cloneMercurialOpenJDKRepo +performMergeIntoDevFromMaster diff --git a/git-hg/setup.sh b/git-hg/setup.sh deleted file mode 100755 index c133aa725..000000000 --- a/git-hg/setup.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -set -euo pipefail -rm -rf "$WORKSPACE/combined" "$WORKSPACE/hg" -mkdir "$WORKSPACE/combined" -mkdir "$WORKSPACE/hg" -cd "$WORKSPACE/combined" || exit 1 -git init -git checkout -b root-commit || exit 1 -git remote add github git@github.com:AdoptOpenJDK/openjdk-"${1}".git -cd - || exit 1 -bash add-branch-without-modules.sh "${2:-jdk/jdk}" diff --git a/git-hg/setup8.sh b/git-hg/setup8.sh deleted file mode 100755 index 7d5f08e80..000000000 --- a/git-hg/setup8.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -set -euo pipefail -rm -rf "$WORKSPACE/combined" "$WORKSPACE/hg" -mkdir "$WORKSPACE/combined" -mkdir "$WORKSPACE/hg" -cd "$WORKSPACE/combined" || exit 1 -git init -git checkout -b root-commit || exit 1 -git remote add github git@github.com:AdoptOpenJDK/openjdk-jdk8u.git || exit 1 -cd - || exit 1 -bash add-branch.sh jdk8u/jdk8u diff --git a/git-hg/setup9.sh b/git-hg/setup9.sh deleted file mode 100755 index 068a7e96e..000000000 --- a/git-hg/setup9.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -set -euo pipefail -rm -rf "$WORKSPACE/combined" "$WORKSPACE/hg" -mkdir "$WORKSPACE/combined" -mkdir "$WORKSPACE/hg" -cd "$WORKSPACE/combined" || exit 1 -git init -git checkout -b root-commit || exit 1 -git remote add github git@github.com:AdoptOpenJDK/openjdk-jdk9.git -cd - || exit 1 -bash add-branch.sh jdk9/jdk9 diff --git a/git-hg/setupAmber.sh b/git-hg/setupAmber.sh deleted file mode 100644 index 50207133b..000000000 --- a/git-hg/setupAmber.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -set -euo pipefail -rm -rf "$WORKSPACE/combined" "$WORKSPACE/hg" -mkdir "$WORKSPACE/combined" -mkdir "$WORKSPACE/hg" -cd "$WORKSPACE/combined" || exit 1 -git init -git checkout -b root-commit || exit 1 -git remote add github git@github.com:AdoptOpenJDK/openjdk-amber.git -cd - || exit 1 -bash add-branch-without-modules.sh amber/amber raw-string-literal diff --git a/git-hg/update-without-modules.sh b/git-hg/update-without-modules.sh deleted file mode 100755 index d335540e1..000000000 --- a/git-hg/update-without-modules.sh +++ /dev/null @@ -1,58 +0,0 @@ -#!/bin/bash -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - - -echo "Import common functionality" -# shellcheck disable=SC1091 -source import-common.sh - -echo "Enter hg" -cd hg || exit 1 - -# shellcheck disable=SC2035 -bpaths=${1:-$(ls -d -1 */*)} # maintain backward compatibility - -for bpath in $bpaths -do - bpathAsArray=(${bpath/\// }) # for e.g. jdk10/jdk10 or jdk/jdk or openjfx/jfx-dev/rt, becomes: - hg_root_forest=${bpathAsArray[0]} # jdk10 or jdk or openjfx - hg_repo=${bpathAsArray[1]} # jdk10 or jdk or jfx-dev/rt - - pushd "$hg_root_forest/$hg_repo/root" - echo "Update $hg_root_forest/$hg_repo -> (root)" - git hg fetch "http://hg.openjdk.java.net/$hg_root_forest/$hg_repo" - git hg pull "http://hg.openjdk.java.net/$hg_root_forest/$hg_repo" - popd - - echo "Exit hg" - - echo "Enter ../combined" - cd ../combined || exit 1 - - echo "Check out master" - git checkout master || exit 1 - - echo "Fetch (root) of the mercurial forest/repo" - git fetch "imports/$hg_root_forest/$hg_repo/root" || exit 1 - - echo "Merge (root)" - git merge "imports/$hg_root_forest/$hg_repo/root/master" -m "Merge from (root)" --no-ff || exit 1 - - echo "Push to GitHub" - git push github master --tags - - echo "Exit back to ../hg" - cd ../hg || exit 1 -done diff --git a/git-hg/updateforesttags.sh b/git-hg/updateforesttags.sh deleted file mode 100755 index 1e4daded4..000000000 --- a/git-hg/updateforesttags.sh +++ /dev/null @@ -1,151 +0,0 @@ -#!/bin/bash -# -# =========================================================================== -# (c) Copyright IBM Corp. 2018 All Rights Reserved -# =========================================================================== -# -# This code is free software; you can redistribute it and/or modify it -# under the terms of the GNU General Public License version 2 only, as -# published by the Free Software Foundation. -# -# This code is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License -# version 2 for more details (a copy is included in the LICENSE file that -# accompanied this code). -# -# You should have received a copy of the GNU General Public License version -# 2 along with this work; if not, see . -# -# =========================================================================== - -if [ $# -lt 1 ]; then - echo Usage: "$0" '[jdk8u|jdk9] (TAGS)' - echo "Version supplied should match a repository in AdoptOpenJDK/openjdk-VERSION" - exit 1 -fi -[ -z "$WORKSPACE" ] || [ ! -d "$WORKSPACE" ] && echo Cannot access \$WORKSPACE: "$WORKSPACE" && exit 2 - -MODULES=(corba langtools jaxp jaxws nashorn jdk hotspot) - -OPENJDK_VERSION="$1" -shift -# shellcheck disable=SC2124 -TAGS="$@" -GITHUB_PROJECT=git@github.com:AdoptOpenJDK -GITHUB_REPO="openjdk-$OPENJDK_VERSION" - -case "$OPENJDK_VERSION" in - jdk8*) HG_REPO=http://hg.openjdk.java.net/jdk8u/jdk8u - [ -z "$TAGS" ] && TAGS="jdk8u144-b34 jdk8u151-b12 jdk8u152-b16 jdk8u161-b12 jdk8u162-b12 jdk8u172-b03 jdk8u172-b11";; - jdk9*) HG_REPO=http://hg.openjdk.java.net/jdk-updates/jdk9u - [ -z "$TAGS" ] && TAGS="jdk-9+181 jdk-9.0.1+11 jdk-9.0.3+9 jdk-9.0.4+11";; - *) Unknown JDK version - only jdk8u and jdk9 are supported; exit 1;; -esac - -# Clean up -rm -rf "${WORKSPACE:?}/$GITHUB_REPO" "${WORKSPACE:?}/openjdk" -mkdir -p "$WORKSPACE/$GITHUB_REPO" "$WORKSPACE/openjdk/mirror" - -git --version || exit 1 -GIT_VERSION=$(git --version | awk '{print$NF}') -GIT_MAJOR_VERSION=$(echo "$GIT_VERSION" | cut -d. -f1) -GIT_MINOR_VERSION=$(echo "$GIT_VERSION" | cut -d. -f2) -[ "$GIT_MAJOR_VERSION" -eq 1 ] && echo I need git version 2.16 or later and you have "$GIT_VERSION" && exit 1 -[ "$GIT_MAJOR_VERSION" -eq 2 ] && [ "$GIT_MINOR_VERSION" -lt 16 ] && echo I need git version 2.16 or later and you have "$GIT_VERSION" && exit 1 - -if ! which git-remote-hg 2>/dev/null; then - echo "I need git-remote-hg and could not find it" - echo "Get it from http://raw.githubusercontent.com/felipec/git-remote-hg/master/git-remote-hg" - exit 1 -fi - -# Clone current AdoptOpenJDK repo -cd "$WORKSPACE/$GITHUB_REPO" || exit 1 -echo "Clone current $GITHUB_REPO" -git clone "$GITHUB_PROJECT/$GITHUB_REPO.git" -cd "$GITHUB_REPO" || exit -git fetch --tags -OLDTAG=$(git describe --abbrev=0 --tags) -echo "Current openjdk level is $OLDTAG" - -# Clone current openjdk from Mercurial -echo "Get base openjdk repository" -cd "$WORKSPACE/openjdk/mirror" || exit 1 -git init -git clone --bare "hg::${HG_REPO}" - -cd "$OPENJDK_VERSION.git" || exit 1 - -git filter-branch -f --index-filter 'git rm -r -f -q --cached --ignore-unmatch .hg .hgignore .hgtags get_source.sh' --prune-empty --tag-name-filter cat -- --all - -cd .. || exit -git pull "$OPENJDK_VERSION" -git fetch --tags "$OPENJDK_VERSION" -for NEWTAG in $TAGS ; do - cd "$WORKSPACE/openjdk/mirror" || exit 1 - git reset --hard "$NEWTAG" - - echo "$(date +%T)": "Updating master branch for $NEWTAG" - cd "$WORKSPACE/$GITHUB_REPO/$GITHUB_REPO" || exit 1 - git checkout master - git fetch "$WORKSPACE/openjdk/mirror" - git merge --allow-unrelated-histories -m "Merge base $NEWTAG" FETCH_HEAD - - for module in "${MODULES[@]}" - do - if [ ! -d "$WORKSPACE/openjdk/$module" ]; then - rm -rf "$WORKSPACE/openjdk/$module" - mkdir "$WORKSPACE/openjdk/$module" - cd "$WORKSPACE/openjdk/$module" || exit 1 - git init - echo "$(date +%T)": "Clone $module" - git clone --bare "hg::${HG_REPO}/$module" || exit 1 - echo "$(date +%T)": "GIT filter on $module" - cd "$module.git" || exit - git filter-branch -f --index-filter "git rm -f -q --cached --ignore-unmatch .hgignore .hgtags && git ls-files -s | sed \"s|\t\\\"*|&$module/|\" | GIT_INDEX_FILE=\$GIT_INDEX_FILE.new git update-index --index-info && mv \"\$GIT_INDEX_FILE.new\" \"\$GIT_INDEX_FILE\"" --prune-empty --tag-name-filter cat -- --all - cd .. || exit - fi - echo "$(date +%T)": "GIT pull/reset on $module at $NEWTAG" - cd "$WORKSPACE/openjdk/$module" || exit 1 - git pull "$module" - git fetch --tags "$module" - git reset --hard "$NEWTAG" - cd "$WORKSPACE/$GITHUB_REPO/$GITHUB_REPO" || exit 1 - git fetch "$WORKSPACE/openjdk/$module" - echo "$(date +%T)": GIT filter on "$module" - if ! git merge --allow-unrelated-histories -m "Merge $module at $NEWTAG" FETCH_HEAD; then - if ! tty; then - echo "Aborting - not running on a real tty therefore cannot allow manual intervention" - exit 10 - else - echo "Please resolve them in another window then press return to continue" - read -r _ - fi - echo Please resolve the conflicts above in "$WORKSPACE/$GITHUB_REPO/$GITHUB_REPO", and press return to continue - read -r _ - fi - done - cd "$WORKSPACE/$GITHUB_REPO/$GITHUB_REPO" || exit 1 - git push origin master - - echo "Pulling in changes to $GITHUB_REPO branch" - git checkout master - git fetch origin master - if ! git merge --allow-unrelated-histories -m "Merge $NEWTAG into $GITHUB_REPO" FETCH_HEAD; then - echo Conflict resolution needed in "$WORKSPACE/$GITHUB_REPO/$GITHUB_REPO" - if ! tty; then - echo "Aborting - not running on a real tty therefore cannot allow manual intervention" - exit 10 - else - echo "Please resolve them in another window then press return to continue" - read -r _ - fi - fi - [ "$NEWTAG" != "HEAD" ] && git tag -f -a "$NEWTAG" -m "Merge $NEWTAG into master" - echo Deleting the old version of the tag from the server if it is present or push will fail - # shellcheck disable=SC2015 - [ "$NEWTAG" != "HEAD" ] && git push origin :refs/tags/"$NEWTAG" || true - [ "$NEWTAG" == "HEAD" ] && git push origin master - [ "$NEWTAG" != "HEAD" ] && git push origin master --tags -done