publish to sonatype (#10)

Author: fqaiser94

.github/workflows/cicd.yml

@@ -13,6 +13,9 @@ jobs:
         uses: lokkju/[email protected]
         with:
           commands: + package
+      - name: copy and rename package for python tests
+        run: |
+          cp target/scala-2.11/*.jar python/mse.jar
       - name: set up python3
         uses: actions/setup-python@v1
         with:
@@ -31,9 +34,12 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v1
-      - name: sbt deploy
-        uses: lokkju/[email protected]
-        with:
-          commands: + publish
+      - uses: olafurpg/setup-scala@v2
+      - uses: olafurpg/setup-gpg@v2
+      - name: publish to sonatype
+        run: csbt ci-release
         env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          PGP_PASSPHRASE: ${{ secrets.PGP_PASSPHRASE }}
+          PGP_SECRET: ${{ secrets.PGP_SECRET }}
+          SONATYPE_PASSWORD: ${{ secrets.SONATYPE_PASSWORD }}
+          SONATYPE_USERNAME: ${{ secrets.SONATYPE_USERNAME }}

.gitignore

@@ -4,3 +4,4 @@ project/target
 target/
 /python/venv/
 /mse.iml
+/python/mse.jar

README.md

@@ -55,7 +55,7 @@ spark-shell --jars mse_2.11-0.1.4.jar
 To bring in to scope the (implicit) Column methods, use:   
 
 ```scala
-import com.mse.column.methods._
+import com.github.fqaiser94.mse.methods._
 ```
 
 You can now use these methods to manipulate fields in a top-level StructType column: 
@@ -279,8 +279,8 @@ spark-shell --jars mse_2.11-0.1.4.jar --packages "za.co.absa:spark-hofs_2.11:0.4
 ```scala
 import org.apache.spark.sql._
 import org.apache.spark.sql.types._
-import com.mse.column.methods._
 import za.co.absa.spark.hofs._
+import com.github.fqaiser94.mse.methods._
 
 // Generate some example data
 val arrayOfStructs = spark.createDataFrame(sc.parallelize(

build.sbt

@@ -1,7 +1,5 @@
 name := "mse"
 
-version := "0.1.5"
-
 scalaVersion := "2.12.10"
 
 crossScalaVersions := Seq("2.11.12", "2.12.10")
@@ -17,11 +15,10 @@ libraryDependencies ++= Seq(
   "org.scalacheck" %% "scalacheck" % "1.14.1" % Test
 )
 
-licenses := Seq("Apache License 2.0" -> url("http://www.apache.org/licenses/LICENSE-2.0.html"))
-
 parallelExecution in Test := false
 
-ThisBuild / githubOwner := "fqaiser94"
-ThisBuild / githubRepository := "mse"
-ThisBuild / githubUser := sys.env.getOrElse("GITHUB_USER", "")
-ThisBuild / githubTokenSource := Some(TokenSource.Environment("GITHUB_TOKEN"))
+inThisBuild(List(
+  organization := "com.github.fqaiser94",
+  homepage := Some(url("https://github.com/fqaiser94/mse")),
+  licenses := Seq("Apache License 2.0" -> url("http://www.apache.org/licenses/LICENSE-2.0.html"))
+))

project/plugins.sbt

@@ -1 +1 @@
-addSbtPlugin("com.codecommit" % "sbt-github-packages" % "0.2.1")
+addSbtPlugin("com.geirsson" % "sbt-ci-release" % "1.5.0")

python/mse/methods.py

@@ -8,7 +8,7 @@ def __withField(self: Column, fieldName: str, fieldValue: Column):
     If schema contains multiple fields with fieldName, they will all be replaced with fieldValue.
     """
     sc = SparkContext._active_spark_context
-    _columnWithCustomMethods = sc._jvm.com.mse.column.methods.ColumnWithCustomMethods(self._jc)
+    _columnWithCustomMethods = sc._jvm.com.github.fqaiser94.mse.methods.ColumnWithCustomMethods(self._jc)
     _column = _columnWithCustomMethods.withField(fieldName, fieldValue._jc)
     return Column(_column)
 
@@ -20,7 +20,7 @@ def __dropFields(self: Column, *fieldNames: str):
     If schema contains multiple fields matching any one of the given fieldNames, they will all be dropped.
     """
     sc = SparkContext._active_spark_context
-    _columnWithCustomMethods = sc._jvm.com.mse.column.methods.ColumnWithCustomMethods(self._jc)
+    _columnWithCustomMethods = sc._jvm.com.github.fqaiser94.mse.methods.ColumnWithCustomMethods(self._jc)
     _fieldNames = sc._jvm.PythonUtils.toSeq(fieldNames)
     _column = _columnWithCustomMethods.dropFields(_fieldNames)
     return Column(_column)
@@ -33,7 +33,7 @@ def __withFieldRenamed(self: Column, existingFieldName: str, newFieldName: str):
     If schema contains multiple fields with existingFieldName, they will all be renamed to newFieldName.
     """
     sc = SparkContext._active_spark_context
-    _columnWithCustomMethods = sc._jvm.com.mse.column.methods.ColumnWithCustomMethods(self._jc)
+    _columnWithCustomMethods = sc._jvm.com.github.fqaiser94.mse.methods.ColumnWithCustomMethods(self._jc)
     _column = _columnWithCustomMethods.withFieldRenamed(existingFieldName, newFieldName)
     return Column(_column)
 
@@ -52,6 +52,6 @@ def add_struct_field(nestedStruct: str, fieldName: str, fieldValue: Column):
     :return: a copy the top-level struct column (a) with field added/replaced.
     """
     sc = SparkContext._active_spark_context
-    _add_struct_field = sc._jvm.com.mse.column.methods.add_struct_field
+    _add_struct_field = sc._jvm.com.github.fqaiser94.mse.methods.add_struct_field
     _column = _add_struct_field(nestedStruct, fieldName, fieldValue._jc)
     return Column(_column)

python/testing/ReusedPySparkTestCase.py

@@ -28,7 +28,7 @@ def conf(cls):
         Override this in subclasses to supply a more specific conf
         """
 
-        return SparkConf().set("spark.driver.extraClassPath", "./../target/scala-2.11/mse_2.11-0.1.5.jar")
+        return SparkConf().set("spark.driver.extraClassPath", "./mse.jar")
 
     @classmethod
     def setUpClass(cls):

src/main/scala/com/mse/column/methods.scala renamed to src/main/scala/com/github/fqaiser94/mse/methods.scala

@@ -1,4 +1,4 @@
-package com.mse.column
+package com.github.fqaiser94.mse
 
 import org.apache.spark.sql.{Column, ColumnName}
 import org.apache.spark.sql.catalyst.analysis.UnresolvedAttribute

src/test/scala/com/mse/column/QueryTester.scala renamed to src/test/scala/com/github/fqaiser94/mse/QueryTester.scala

@@ -1,4 +1,4 @@
-package com.mse.column
+package com.github.fqaiser94.mse
 
 import org.apache.spark.sql.test.SharedSparkSession
 import org.apache.spark.sql.types.StructType

src/test/scala/com/mse/column/add_struct_fieldTest.scala renamed to src/test/scala/com/github/fqaiser94/mse/add_struct_fieldTest.scala

@@ -1,6 +1,6 @@
-package com.mse.column
+package com.github.fqaiser94.mse
 
-import com.mse.column.methods._
+import methods._
 import org.apache.spark.sql.functions._
 import org.apache.spark.sql.test.SharedSparkSession
 import org.apache.spark.sql.types._