Skip to content

Commit 7c5ae09

Browse files
SemyonSinchenkoKimahriman
SemyonSinchenko
and
Kimahriman
authored
fix: resolve gf-connect is not published (#645)
* Try to fix connect publishing * Refactoring to the right multi-project sbt structure * Fix Scala CI * fix scala-ci.yml * Update .github/workflows/scala-ci.yml Co-authored-by: Adam Binford <[email protected]> * Update .github/workflows/scala-ci.yml Co-authored-by: Adam Binford <[email protected]> * Update .github/workflows/scala-ci.yml Co-authored-by: Adam Binford <[email protected]> * Update .github/workflows/scala-ci.yml Co-authored-by: Adam Binford <[email protected]> * refactoring according comments * scalafix * decrease java version in publish --------- Co-authored-by: Adam Binford <[email protected]>
1 parent ff7a20d commit 7c5ae09

File tree

56 files changed

+14
-19
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

56 files changed

+14
-19
lines changed

‎.github/workflows/scala-ci.yml‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -30,9 +30,9 @@ jobs:
3030
~/.ivy2/cache
3131
key: sbt-ivy-cache-spark-${{ matrix.spark-version}}-java-${{ matrix.java-version }}
3232
- name: Check scalafmt
33-
run: build/sbt -Dspark.version=${{ matrix.spark-version }} root/scalafmtCheckAll connect/scalafmtCheckAll
33+
run: build/sbt -Dspark.version=${{ matrix.spark-version }} scalafmtCheckAll
3434
- name: Check scalastyle
35-
run: build/sbt -Dspark.version=${{ matrix.spark-version }} root/scalafixAll connect/scalafixAll
35+
run: build/sbt -Dspark.version=${{ matrix.spark-version }} "scalafixAll --check"
3636
- name: Build and Test
37-
run: build/sbt -v -Dspark.version=${{ matrix.spark-version }} coverage +test coverageReport "project connect" coverage +test coverageReport
37+
run: build/sbt -v -Dspark.version=${{ matrix.spark-version }} coverage +test coverageReport
3838
- uses: codecov/codecov-action@v3

‎.github/workflows/scala-publish.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ jobs:
2121
matrix:
2222
include:
2323
- spark-version: 3.5.5
24-
java-version: 11
24+
java-version: 8
2525
- spark-version: 4.0.0
2626
java-version: 17
2727
runs-on: ubuntu-latest
@@ -38,7 +38,7 @@ jobs:
3838
distribution: 'zulu'
3939

4040
- name: Run the release of core
41-
run: build/sbt -Dspark.version=${{ matrix.spark-version }} +ci-release "project connect" +ci-release
41+
run: build/sbt -Dspark.version=${{ matrix.spark-version }} ci-release
4242
env:
4343
PGP_PASSPHRASE: ${{ secrets.PGP_PASSPHRASE }}
4444
PGP_SECRET: ${{ secrets.PGP_SECRET }}

‎build.sbt‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -100,7 +100,7 @@ lazy val commonSetting = Seq(
100100
else
101101
"-Wunused:imports"))
102102

103-
lazy val root = (project in file("."))
103+
lazy val core = (project in file("core"))
104104
.settings(
105105
commonSetting,
106106
name := "graphframes",
@@ -122,8 +122,8 @@ lazy val root = (project in file("."))
122122
Compile / packageDoc / publishArtifact := true,
123123
Compile / packageSrc / publishArtifact := true)
124124

125-
lazy val connect = (project in file("graphframes-connect"))
126-
.dependsOn(root)
125+
lazy val connect = (project in file("connect"))
126+
.dependsOn(core)
127127
.settings(
128128
name := s"graphframes-connect",
129129
moduleName := s"${name.value}-spark${sparkMajorVer}",

graphframes-connect/src/main/protobuf/graphframes.proto renamed to connect/src/main/protobuf/graphframes.proto

File renamed without changes.

graphframes-connect/src/main/scala-spark-3/org/apache/spark/sql/graphframes/GraphFramesConnect.scala renamed to connect/src/main/scala-spark-3/org/apache/spark/sql/graphframes/GraphFramesConnect.scala

File renamed without changes.

graphframes-connect/src/main/scala-spark-4/org/apache/spark/sql/graphframes/GraphFramesConnect.scala renamed to connect/src/main/scala-spark-4/org/apache/spark/sql/graphframes/GraphFramesConnect.scala

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,12 @@
11
package org.apache.spark.sql.graphframes
22

3-
import java.util.Optional
4-
5-
import org.graphframes.connect.proto.GraphFramesAPI
6-
3+
import com.google.protobuf
74
import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
85
import org.apache.spark.sql.connect.planner.SparkConnectPlanner
96
import org.apache.spark.sql.connect.plugin.RelationPlugin
7+
import org.graphframes.connect.proto.GraphFramesAPI
108

11-
import com.google.protobuf
9+
import java.util.Optional
1210

1311
class GraphFramesConnect extends RelationPlugin {
1412
override def transform(

graphframes-connect/src/main/scala/org/apache/spark/sql/graphframes/GraphFramesConnectUtils.scala renamed to connect/src/main/scala/org/apache/spark/sql/graphframes/GraphFramesConnectUtils.scala

File renamed without changes.

src/main/scala-spark-3/org/apache/spark/sql/graphframes/SparkShims.scala renamed to core/src/main/scala-spark-3/org/apache/spark/sql/graphframes/SparkShims.scala

File renamed without changes.

src/main/scala-spark-4/org/apache/spark/sql/graphframes/SparkShims.scala renamed to core/src/main/scala-spark-4/org/apache/spark/sql/graphframes/SparkShims.scala

File renamed without changes.

src/main/scala/org/apache/spark/sql/graphframes/GraphFramesConf.scala renamed to core/src/main/scala/org/apache/spark/sql/graphframes/GraphFramesConf.scala

File renamed without changes.

0 commit comments

Comments
 (0)