Skip to content

Commit ed828e3

Browse files
SemyonSinchenkoSemyonSinchenko
authored
chore: drop sbt-spark-packages && update plugins (#487)
* Drop sbt-spark-packages && update plugins + drop sbt-spark-packages + rewrite build.sbt in the modern way + update plugins * Update sbt-launch-lib like in Apache Spark * Explicit crossScalaVersions to make CI works * Fix assembly && 2.13 build * Fix syntax * Downgrade scoverage to work with 2.13.8 * Spark version like in CI by default * Sbt 1.9.3 (like in Spark) * Downgrade scoverage * An attempt to fix ivy2 cache problem * Cache problem * Downgrade scoverage * Return an old scala ver to CI * 2.12.18 in CI * Update the plugin * Update the version * Try a workaround * 2.0.10 scoverage
1 parent bc487ef commit ed828e3

File tree

7 files changed

+123
-123
lines changed

7 files changed

+123
-123
lines changed

‎.github/workflows/scala-ci.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ jobs:
99
- spark-version: 3.5.4
1010
scala-version: 2.13.8
1111
- spark-version: 3.5.4
12-
scala-version: 2.12.12
12+
scala-version: 2.12.18
1313
runs-on: ubuntu-22.04
1414
env:
1515
# fixing this error after tests success: sbt.ForkMain failed with exit code 134

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ lib_managed/
1616
src_managed/
1717
project/boot/
1818
project/plugins/project/
19+
.bsp
1920

2021
# intellij
2122
.idea/

‎build.sbt‎

Lines changed: 84 additions & 83 deletions
Original file line numberDiff line numberDiff line change
@@ -1,92 +1,93 @@
1-
// Your sbt build file. Guides on how to write one can be found at
2-
// http://www.scala-sbt.org/0.13/docs/index.html
3-
41
import ReleaseTransformations._
52

6-
val sparkVer = sys.props.getOrElse("spark.version", "3.5.3")
7-
val sparkBranch = sparkVer.substring(0, 3)
8-
val defaultScalaVer = sparkBranch match {
3+
lazy val sparkVer = sys.props.getOrElse("spark.version", "3.5.4")
4+
lazy val sparkBranch = sparkVer.substring(0, 3)
5+
lazy val defaultScalaVer = sparkBranch match {
96
case "3.5" => "2.12.18"
107
case _ => throw new IllegalArgumentException(s"Unsupported Spark version: $sparkVer.")
118
}
12-
val scalaVer = sys.props.getOrElse("scala.version", defaultScalaVer)
13-
val defaultScalaTestVer = scalaVer match {
9+
lazy val scalaVer = sys.props.getOrElse("scala.version", defaultScalaVer)
10+
lazy val defaultScalaTestVer = scalaVer match {
1411
case s if s.startsWith("2.12") || s.startsWith("2.13") => "3.0.8"
1512
}
1613

17-
sparkVersion := sparkVer
18-
19-
scalaVersion := scalaVer
20-
21-
name := "graphframes"
22-
23-
spName := "graphframes/graphframes"
24-
25-
organization := "org.graphframes"
26-
27-
version := (version in ThisBuild).value + s"-spark$sparkBranch"
28-
29-
isSnapshot := version.value.contains("SNAPSHOT")
30-
31-
// All Spark Packages need a license
32-
licenses := Seq("Apache-2.0" -> url("http://opensource.org/licenses/Apache-2.0"))
33-
34-
spAppendScalaVersion := true
35-
36-
// Add Spark components this package depends on, e.g, "mllib", ....
37-
sparkComponents ++= Seq("graphx", "sql", "mllib")
38-
39-
// uncomment and change the value below to change the directory where your zip artifact will be created
40-
// spDistDirectory := target.value
41-
42-
// add any Spark Package dependencies using spDependencies.
43-
// e.g. spDependencies += "databricks/spark-avro:0.1"
44-
45-
libraryDependencies += "org.slf4j" % "slf4j-api" % "1.7.16"
46-
47-
libraryDependencies += "org.scalatest" %% "scalatest" % defaultScalaTestVer % "test"
48-
49-
libraryDependencies += "com.github.zafarkhaja" % "java-semver" % "0.9.0" % "test" // MIT license
50-
51-
parallelExecution := false
52-
53-
scalacOptions ++= Seq("-deprecation", "-feature")
54-
55-
scalacOptions in (Compile, doc) ++= Seq(
56-
"-groups",
57-
"-implicits",
58-
"-skip-packages", Seq("org.apache.spark").mkString(":"))
59-
60-
scalacOptions in (Test, doc) ++= Seq("-groups", "-implicits")
61-
62-
// This fixes a class loader problem with scala.Tuple2 class, scala-2.11, Spark 2.x
63-
fork in Test := true
64-
65-
// This and the next line fix a problem with forked run: https://github.com/scalatest/scalatest/issues/770
66-
javaOptions in Test ++= Seq(
67-
"-XX:+IgnoreUnrecognizedVMOptions",
68-
"-Xmx2048m",
69-
"-XX:ReservedCodeCacheSize=384m",
70-
"-XX:MaxMetaspaceSize=384m",
71-
"--add-opens=java.base/sun.nio.ch=ALL-UNNAMED",
72-
"--add-opens=java.base/java.lang=ALL-UNNAMED"
73-
)
74-
75-
concurrentRestrictions in Global := Seq(
76-
Tags.limitAll(1))
77-
78-
autoAPIMappings := true
79-
80-
coverageHighlighting := false
81-
82-
// We only use sbt-release to update version numbers.
83-
releaseProcess := Seq[ReleaseStep](
84-
inquireVersions,
85-
setReleaseVersion,
86-
commitReleaseVersion,
87-
tagRelease,
88-
setNextVersion,
89-
commitNextVersion
90-
)
14+
ThisBuild / version := {
15+
val baseVersion = (ThisBuild / version).value
16+
s"${baseVersion}-spark${sparkBranch}"
17+
}
9118

92-
credentials += Credentials(Path.userHome / ".ivy2" / ".sbtcredentials")
19+
ThisBuild / scalaVersion := scalaVer
20+
ThisBuild / organization := "org.graphframes"
21+
ThisBuild / crossScalaVersions := Seq("2.12.18", "2.13.8")
22+
23+
lazy val root = (project in file("."))
24+
.settings(
25+
name := "graphframes",
26+
27+
// Replace spark-packages plugin functionality with explicit dependencies
28+
libraryDependencies ++= Seq(
29+
"org.apache.spark" %% "spark-graphx" % sparkVer % "provided" cross CrossVersion.for3Use2_13,
30+
"org.apache.spark" %% "spark-sql" % sparkVer % "provided" cross CrossVersion.for3Use2_13,
31+
"org.apache.spark" %% "spark-mllib" % sparkVer % "provided" cross CrossVersion.for3Use2_13,
32+
"org.slf4j" % "slf4j-api" % "1.7.16",
33+
"org.scalatest" %% "scalatest" % defaultScalaTestVer % Test,
34+
"com.github.zafarkhaja" % "java-semver" % "0.9.0" % Test
35+
),
36+
37+
licenses := Seq("Apache-2.0" -> url("http://opensource.org/licenses/Apache-2.0")),
38+
39+
// Modern way to set Scala options
40+
Compile / scalacOptions ++= Seq("-deprecation", "-feature"),
41+
42+
Compile / doc / scalacOptions ++= Seq(
43+
"-groups",
44+
"-implicits",
45+
"-skip-packages", Seq("org.apache.spark").mkString(":")
46+
),
47+
48+
Test / doc / scalacOptions ++= Seq("-groups", "-implicits"),
49+
50+
// Test settings
51+
Test / fork := true,
52+
Test / parallelExecution := false,
53+
54+
Test / javaOptions ++= Seq(
55+
"-XX:+IgnoreUnrecognizedVMOptions",
56+
"-Xmx2048m",
57+
"-XX:ReservedCodeCacheSize=384m",
58+
"-XX:MaxMetaspaceSize=384m",
59+
"--add-opens=java.base/sun.nio.ch=ALL-UNNAMED",
60+
"--add-opens=java.base/java.lang=ALL-UNNAMED"
61+
),
62+
63+
// Global settings
64+
Global / concurrentRestrictions := Seq(
65+
Tags.limitAll(1)
66+
),
67+
68+
autoAPIMappings := true,
69+
70+
coverageHighlighting := false,
71+
72+
// Release settings
73+
releaseProcess := Seq[ReleaseStep](
74+
inquireVersions,
75+
setReleaseVersion,
76+
commitReleaseVersion,
77+
tagRelease,
78+
setNextVersion,
79+
commitNextVersion
80+
),
81+
82+
// Assembly settings
83+
assembly / test := {}, // No tests in assembly
84+
assembly / assemblyMergeStrategy := {
85+
case PathList("META-INF", xs @ _*) => MergeStrategy.discard
86+
case x if x.endsWith("module-info.class") => MergeStrategy.discard
87+
case x =>
88+
val oldStrategy = (assembly / assemblyMergeStrategy).value
89+
oldStrategy(x)
90+
},
91+
92+
credentials += Credentials(Path.userHome / ".ivy2" / ".sbtcredentials")
93+
)

‎build/sbt-launch-lib.bash‎

Lines changed: 25 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77
# TODO - Should we merge the main SBT script with this library?
88

99
if test -z "$HOME"; then
10-
declare -r script_dir="$(dirname $script_path)"
10+
declare -r script_dir="$(dirname "$script_path")"
1111
else
1212
declare -r script_dir="$HOME/.sbt"
1313
fi
@@ -17,6 +17,7 @@ declare -a java_args
1717
declare -a scalac_args
1818
declare -a sbt_commands
1919
declare -a maven_profiles
20+
declare sbt_default_mem=4096
2021

2122
if test -x "$JAVA_HOME/bin/java"; then
2223
echo -e "Using $JAVA_HOME as default JAVA_HOME."
@@ -37,32 +38,36 @@ dlog () {
3738
}
3839

3940
acquire_sbt_jar () {
40-
SBT_VERSION=`awk -F "=" '/sbt\\.version/ {print $2}' ./project/build.properties`
41-
URL1=https://repo.scala-sbt.org/scalasbt/ivy-releases/org.scala-sbt/sbt-launch/${SBT_VERSION}/sbt-launch.jar
41+
SBT_VERSION=`awk -F "=" '/sbt\.version/ {print $2}' ./project/build.properties`
42+
# DEFAULT_ARTIFACT_REPOSITORY env variable can be used to only fetch
43+
# artifacts from internal repos only.
44+
# Ex:
45+
# DEFAULT_ARTIFACT_REPOSITORY=https://artifacts.internal.com/libs-release/
46+
URL1=${DEFAULT_ARTIFACT_REPOSITORY:-https://repo1.maven.org/maven2/}org/scala-sbt/sbt-launch/${SBT_VERSION}/sbt-launch-${SBT_VERSION}.jar
4247
JAR=build/sbt-launch-${SBT_VERSION}.jar
4348

4449
sbt_jar=$JAR
4550

4651
if [[ ! -f "$sbt_jar" ]]; then
4752
# Download sbt launch jar if it hasn't been downloaded yet
48-
if [ ! -f ${JAR} ]; then
53+
if [ ! -f "${JAR}" ]; then
4954
# Download
5055
printf "Attempting to fetch sbt\n"
51-
JAR_DL=${JAR}.part
52-
if hash curl 2>/dev/null; then
56+
JAR_DL="${JAR}.part"
57+
if [ $(command -v curl) ]; then
5358
curl --fail --location --silent ${URL1} > "${JAR_DL}" &&\
5459
mv "${JAR_DL}" "${JAR}"
55-
elif hash wget 2>/dev/null; then
60+
elif [ $(command -v wget) ]; then
5661
wget --quiet ${URL1} -O "${JAR_DL}" &&\
5762
mv "${JAR_DL}" "${JAR}"
5863
else
59-
printf "You do not have curl or wget installed, please install sbt manually from http://www.scala-sbt.org/\n"
64+
printf "You do not have curl or wget installed, please install sbt manually from https://www.scala-sbt.org/\n"
6065
exit -1
6166
fi
6267
fi
63-
if [ ! -f ${JAR} ]; then
68+
if [ ! -f "${JAR}" ]; then
6469
# We failed to download
65-
printf "Our attempt to download sbt locally to ${JAR} failed. Please install sbt manually from http://www.scala-sbt.org/\n"
70+
printf "Our attempt to download sbt locally to ${JAR} failed. Please install sbt manually from https://www.scala-sbt.org/\n"
6671
exit -1
6772
fi
6873
printf "Launching sbt from ${JAR}\n"
@@ -82,7 +87,7 @@ execRunner () {
8287
echo ""
8388
}
8489

85-
exec "$@"
90+
"$@"
8691
}
8792

8893
addJava () {
@@ -105,27 +110,27 @@ addResidual () {
105110
residual_args=( "${residual_args[@]}" "$1" )
106111
}
107112
addDebugger () {
108-
addJava "-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=$1"
113+
addJava "-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=$1"
109114
}
110115

111116
# a ham-fisted attempt to move some memory settings in concert
112117
# so they need not be dicked around with individually.
113118
get_mem_opts () {
114-
local mem=${1:-2048}
115-
local perm=$(( $mem / 4 ))
116-
(( $perm > 256 )) || perm=256
117-
(( $perm < 4096 )) || perm=4096
118-
local codecache=$(( $perm / 2 ))
119+
local mem=${1:-$sbt_default_mem}
120+
local codecache=$(( $mem / 8 ))
121+
(( $codecache > 128 )) || codecache=128
122+
(( $codecache < 2048 )) || codecache=2048
119123

120-
echo "-Xms${mem}m -Xmx${mem}m -XX:MaxMetaspaceSize=${perm}m -XX:ReservedCodeCacheSize=${codecache}m"
124+
echo "-Xms${mem}m -Xmx${mem}m -XX:ReservedCodeCacheSize=${codecache}m"
121125
}
122126

123127
require_arg () {
124128
local type="$1"
125129
local opt="$2"
126130
local arg="$3"
127131
if [[ -z "$arg" ]] || [[ "${arg:0:1}" == "-" ]]; then
128-
die "$opt requires <$type> argument"
132+
echo "$opt requires <$type> argument" 1>&2
133+
exit 1
129134
fi
130135
}
131136

@@ -178,18 +183,11 @@ run() {
178183

179184
# run sbt
180185
execRunner "$java_cmd" \
181-
${SBT_OPTS:-$default_sbt_opts} \
182186
$(get_mem_opts $sbt_mem) \
187+
${SBT_OPTS:-$default_sbt_opts} \
183188
${java_opts} \
184189
${java_args[@]} \
185190
-jar "$sbt_jar" \
186191
"${sbt_commands[@]}" \
187192
"${residual_args[@]}"
188193
}
189-
190-
runAlternateBoot() {
191-
local bootpropsfile="$1"
192-
shift
193-
addJava "-Dsbt.boot.properties=$bootpropsfile"
194-
run $@
195-
}

‎project/build.properties‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,2 @@
11
// This file should only contain the version of sbt to use.
2-
sbt.version=0.13.18
2+
sbt.version=1.9.3

‎project/plugins.sbt‎

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
1-
// You may use this file to add plugin dependencies for sbt.
2-
resolvers += "Spark Packages repo" at "https://repos.spark-packages.org/"
3-
4-
addSbtPlugin("org.spark-packages" %% "sbt-spark-package" % "0.2.6")
5-
6-
// scalacOptions in (Compile,doc) := Seq("-groups", "-implicits")
7-
8-
addSbtPlugin("org.scoverage" % "sbt-scoverage" % "1.6.1")
9-
10-
addSbtPlugin("com.github.gseitz" % "sbt-release" % "1.0.13")
1+
// https://github.com/scala/bug/issues/12632
2+
// https://github.com/scoverage/sbt-scoverage/issues/475
3+
// Workaround:
4+
ThisBuild / libraryDependencySchemes ++= Seq(
5+
"org.scala-lang.modules" %% "scala-xml" % VersionScheme.Always
6+
)
7+
8+
addSbtPlugin("org.scoverage" % "sbt-scoverage" % "2.0.10")
9+
addSbtPlugin("com.github.sbt" % "sbt-release" % "1.4.0")
10+
addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "2.3.1")

‎version.sbt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
version in ThisBuild := "0.8.4"
1+
ThisBuild / version := "0.8.4"

0 commit comments

Comments
 (0)