Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

dummy #1234

Draft
wants to merge 7 commits into
base: master
Choose a base branch
from
Draft

dummy #1234

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
af3ace2
dummy
prince-cs Sep 17, 2024
9867d53
dummy
prince-cs Sep 18, 2024
50fb601
dummy
prince-cs Sep 19, 2024
87c7787
dummy
prince-cs Sep 19, 2024
5c0a606
dummy
prince-cs Oct 2, 2024
e2ed0b4
pinning libsystemd0 version for debian 12 OS
prince-cs Oct 3, 2024
d2fe646
dummy
prince-cs Oct 7, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 7 additions & 7 deletions BUILD
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,9 @@ test_suite(
":test_cloud_sql_proxy",
":test_dr_elephant",
":test_hive_hcatalog",
":test_hive_llap",
# ":test_hive_llap",
":test_starburst_presto",
":test_spark_rapids",
# ":test_spark_rapids",
"//alluxio:test_alluxio",
"//atlas:test_atlas",
"//bigtable:test_bigtable",
Expand All @@ -18,23 +18,23 @@ test_suite(
"//drill:test_drill",
"//flink:test_flink",
"//ganglia:test_ganglia",
"//gpu:test_gpu",
# "//gpu:test_gpu",
"//h2o:test_h2o",
"//hbase:test_hbase",
"//horovod:test_horovod",
# "//horovod:test_horovod",
"//hue:test_hue",
"//kafka:test_kafka",
"//knox:test_knox",
"//livy:test_livy",
"//mlvm:test_mlvm",
# "//mlvm:test_mlvm",
"//oozie:test_oozie",
"//presto:test_presto",
"//ranger:test_ranger",
"//rapids:test_rapids",
# "//rapids:test_rapids",
"//rstudio:test_rstudio",
"//solr:test_solr",
"//sqoop:test_sqoop",
"//tony:test_tony",
# "//tony:test_tony",
],
)

Expand Down
2 changes: 1 addition & 1 deletion integration_tests/dataproc_test_case.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -117,7 +117,7 @@ def createCluster(self,
args.append("--image-version={}".format(FLAGS.image_version))
if FLAGS.image_version in self.IMAGE_VERSION_2_2:
args.append("--public-ip-address")

#dummy
for i in init_actions:
if "install_gpu_driver.sh" in i or \
"mlvm.sh" in i or "rapids.sh" in i or \
Expand Down
33 changes: 19 additions & 14 deletions rstudio/rstudio.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,11 @@

set -euxo pipefail

function os_id() { grep '^ID=' /etc/os-release | cut -d= -f2 | xargs ; }
function os_version() { grep '^VERSION_ID=' /etc/os-release | cut -d= -f2 | xargs ; }
function is_ubuntu() { [[ "$(os_id)" == 'ubuntu' ]] ; }
function is_ubuntu18() { is_ubuntu && [[ "$(os_version)" == '18.04'* ]] ; }

# Only run on the master node
ROLE="$(/usr/share/google/get_metadata_value attributes/dataproc-role)"

Expand Down Expand Up @@ -103,15 +108,17 @@ function run_with_retries() {
"${cmd[@]}"
}

function install_package() {
local LIBDEFLATE0_URL="http://archive.ubuntu.com/ubuntu/pool/universe/libd/libdeflate/libdeflate0_1.5-3_amd64.deb"
local LIBDEFLATE_DEV_URL="http://archive.ubuntu.com/ubuntu/pool/universe/libd/libdeflate/libdeflate-dev_1.5-3_amd64.deb"
TMP_DIR=$(mktemp -d)
wget -q -P "${TMP_DIR}" "${LIBDEFLATE0_URL}"
dpkg -i "${TMP_DIR}/$(basename "${LIBDEFLATE0_URL}")"
wget -q -P "${TMP_DIR}" "${LIBDEFLATE_DEV_URL}"
dpkg -i "${TMP_DIR}/$(basename "${LIBDEFLATE_DEV_URL}")"
rm -rf "${TMP_DIR}"
function install_libflate_package() {
if is_ubuntu18; then
local LIBDEFLATE0_URL="http://archive.ubuntu.com/ubuntu/pool/universe/libd/libdeflate/libdeflate0_1.5-3_amd64.deb"
local LIBDEFLATE_DEV_URL="http://archive.ubuntu.com/ubuntu/pool/universe/libd/libdeflate/libdeflate-dev_1.5-3_amd64.deb"
TMP_DIR=$(mktemp -d)
wget -q -P "${TMP_DIR}" "${LIBDEFLATE0_URL}"
dpkg -i "${TMP_DIR}/$(basename "${LIBDEFLATE0_URL}")"
wget -q -P "${TMP_DIR}" "${LIBDEFLATE_DEV_URL}"
dpkg -i "${TMP_DIR}/$(basename "${LIBDEFLATE_DEV_URL}")"
rm -rf "${TMP_DIR}"
fi
}

if [[ "${ROLE}" == 'Master' ]]; then
Expand All @@ -138,15 +145,13 @@ if [[ "${ROLE}" == 'Master' ]]; then
if [[ "${OS_ID}" == "ubuntu" ]]; then
curl -fsSL --retry-connrefused --retry 10 --retry-max-time 30 https://cloud.r-project.org/bin/linux/ubuntu/marutter_pubkey.asc | tee -a /etc/apt/trusted.gpg.d/cran_ubuntu_key.asc
fi
if [[ ${OS_ID} == debian ]] && [[ $(echo "${DATAPROC_IMAGE_VERSION}" == "2.2" | bc -l) == 1 ]]; then
apt-get install -y software-properties-common libsystemd0=252.26-1~deb12u2
if is_debian12; then
apt-get install -y libsystemd0=252.26-1~deb12u2 software-properties-common
else
apt-get install -y software-properties-common
fi
add-apt-repository "deb http://cran.r-project.org/bin/linux/${OS_ID} ${OS_CODE}-cran40/"
if [[ ${OS_ID} == ubuntu ]] && [[ $(echo "${DATAPROC_IMAGE_VERSION} < 2.1" | bc -l) == 1 ]]; then
install_package
fi
install_libflate_package
update_apt_get
apt-get install -y r-base r-base-dev gdebi-core

Expand Down