Skip to content

Commit ad2567e

Browse files
author
Christian Herrera
committed
Reorganize lessons
1 parent 1eedf38 commit ad2567e

File tree

116 files changed

+399
-2488
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

116 files changed

+399
-2488
lines changed

‎.github/dependabot.yml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
version: 2
2+
updates:
3+
- package-ecosystem: "github-actions"
4+
directory: "/"
5+
schedule:
6+
interval: "daily"

‎.github/workflows/check-dependencies-updates.yml‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
on:
2+
schedule:
3+
- cron: '0 6 * * 1-5'
4+
5+
name: 🍄 Check dependencies updates
6+
7+
permissions:
8+
contents: write
9+
pull-requests: write
10+
11+
jobs:
12+
scala-steward:
13+
runs-on: ubuntu-22.04
14+
name: Check Scala project dependencies updates with Scala Steward
15+
steps:
16+
- uses: scala-steward-org/scala-steward-action@v2

‎.github/workflows/ci.yml‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
name: CI
2+
3+
on:
4+
push:
5+
branches:
6+
- main
7+
pull_request:
8+
9+
permissions:
10+
contents: read
11+
12+
jobs:
13+
build:
14+
runs-on: ubuntu-latest
15+
steps:
16+
- uses: actions/checkout@v4
17+
- uses: actions/setup-java@v4
18+
with:
19+
distribution: 'zulu'
20+
java-version: '21'
21+
cache: 'sbt'
22+
- name: 👌 Run "pre-push" tasks (compile and style-check)
23+
run: sbt prep
24+
- name: ✅ Run test
25+
run: sbt test

‎.github/workflows/update-github-dependency-graph.yml‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
name: Update GitHub Dependency Graph
2+
3+
on:
4+
push:
5+
branches:
6+
- main
7+
8+
permissions:
9+
contents: write
10+
11+
jobs:
12+
build:
13+
runs-on: ubuntu-latest
14+
steps:
15+
- uses: actions/checkout@v4
16+
- uses: scalacenter/sbt-dependency-submission@v3

‎.scalafmt.conf‎

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,11 @@
1-
version=2.5.2
1+
version = 3.8.2
2+
runner.dialect = scala213
3+
style = default
4+
maxColumn = 120
5+
continuationIndent.callSite = 2
26
align.preset = more
3-
maxColumn = 80
4-
importSelectors = singleLine
7+
runner.optimizer.forceConfigStyleMinArgCount = 1
8+
rewrite.rules = [SortImports]
9+
importSelectors = singleLine
10+
project.excludeFilters = ["target/"]
11+
project.git = true # Only format files tracked by git

‎build.sbt‎

Lines changed: 5 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -1,38 +1,8 @@
1-
scalaVersion := "2.12.12"
2-
version := "0.1.0-SNAPSHOT"
3-
name := "spark-for-programmers-course"
4-
organization := "com.codely"
1+
Settings.settings
52

6-
val sparkVesion = "3.5.0"
3+
libraryDependencies := Dependencies.all
74

8-
libraryDependencies ++= Seq(
9-
"org.apache.spark" %% "spark-core" % sparkVesion,
10-
"org.apache.spark" %% "spark-sql" % sparkVesion,
11-
"org.apache.spark" %% "spark-hive" % sparkVesion,
12-
"org.apache.spark" %% "spark-streaming" % sparkVesion,
13-
"org.apache.spark" %% "spark-sql-kafka-0-10" % sparkVesion,
14-
"io.delta" %% "delta-spark" % "3.1.0",
15-
// "com.amazonaws" % "aws-java-sdk-bundle" % "1.11.375",
16-
"org.apache.hadoop" % "hadoop-aws" % "3.2.2",
17-
"com.rabbitmq" % "amqp-client" % "5.12.0",
18-
"com.typesafe" % "config" % "1.4.1",
19-
//"org.apache.hadoop" % "hadoop-common" % "3.3.1",
20-
"org.scalatest" %% "scalatest" % "3.2.18" % Test,
21-
"org.scalatest" %% "scalatest-flatspec" % "3.2.18" % Test,
22-
"com.dimafeng" %% "testcontainers-scala" % "0.40.12" % Test,
23-
"com.dimafeng" %% "testcontainers-scala-kafka" % "0.40.12" % Test,
24-
"com.dimafeng" %% "testcontainers-scala-postgresql" % "0.41.4" % Test,
25-
"org.postgresql" % "postgresql" % "9.4.1207" % Test,
26-
"org.mockito" %% "mockito-scala" % "1.16.42" % Test
27-
)
28-
29-
assembly / mainClass := Some(
30-
"com.codely.lesson_07_spark_optimize_and_monitoring.video_01__deploy_application.DeploySparkApp"
31-
)
32-
33-
assembly / assemblyMergeStrategy := {
34-
case PathList("META-INF", xs @ _*) => MergeStrategy.discard
35-
case PathList("org", "apache", "spark", "unused", "UnusedStubClass.class") =>
36-
MergeStrategy.first
37-
case _ => MergeStrategy.first
5+
SbtAliases.aliases.flatMap {
6+
case (alias, command) =>
7+
addCommandAlias(alias, command)
388
}

‎doc/hooks/install-hooks.sh‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
#!/bin/sh
2+
3+
cd "$(dirname "$0")/../.."
4+
5+
rm -rf .git/hooks
6+
7+
ln -s ../doc/hooks .git/hooks
8+
sudo chmod -R 777 doc/hooks/*

‎doc/hooks/pre-push‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
#!/bin/bash
2+
3+
# Checks if locally staged changes are formatted properly ignoring non-staged changes.
4+
# Install it with the `install-hooks.sh` script
5+
# Based on: https://gist.github.com/cvogt/2676ed6c6d1abafa3d6a
6+
7+
PATH=$PATH:/usr/local/bin:/usr/local/sbin
8+
9+
echo ""
10+
echo "Running pre-push hook… (you can omit this with --no-verify, but don't)"
11+
12+
echo "* Moving to the project directory…"
13+
_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
14+
DIR=$( echo $_DIR | sed 's/\/.git\/hooks$//' )
15+
16+
echo "* Stashing non-staged changes so we avoid checking them…"
17+
git diff --quiet
18+
hadNoNonStagedChanges=$?
19+
20+
if ! [ $hadNoNonStagedChanges -eq 0 ]
21+
then
22+
git stash --keep-index -u > /dev/null
23+
fi
24+
25+
echo "* Checking pre push conditions ('prep' SBT task)…"
26+
sbt prep > /dev/null
27+
canPush=$?
28+
29+
if [ $canPush -ne 0 ]
30+
then
31+
echo " [KO] Error :("
32+
fi
33+
34+
echo "* Applying the stash with the non-staged changes…"
35+
if ! [ $hadNoNonStagedChanges -eq 0 ]
36+
then
37+
sleep 1 && git stash pop --index > /dev/null & # sleep because otherwise commit fails when this leads to a merge conflict
38+
fi
39+
40+
# Final result
41+
echo ""
42+
43+
if [ $canPush -eq 0 ]
44+
then
45+
echo "[OK] Your code will be pushed young Padawan"
46+
exit 0
47+
else
48+
echo "[KO] Cancelling push due to test code style error (run 'sbt prep' for more information)"
49+
exit 1
50+
fi

‎project/Dependencies.scala‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
import sbt._
2+
3+
object Dependencies {
4+
private val prod = Seq(
5+
"com.github.nscala-time" %% "nscala-time" % "2.32.0",
6+
"com.lihaoyi" %% "pprint" % "0.9.0",
7+
"org.apache.spark" %% "spark-core" % "3.5.0" % Provided,
8+
"org.apache.spark" %% "spark-sql" % "3.5.0" % Provided,
9+
"org.apache.spark" %% "spark-streaming" % "3.5.0",
10+
"org.apache.spark" %% "spark-hive" % "3.5.0",
11+
"io.delta" %% "delta-spark" % "3.1.0",
12+
"org.apache.hadoop" % "hadoop-aws" % "3.2.2"
13+
)
14+
private val test = Seq(
15+
"org.scalatest" %% "scalatest" % "3.2.19",
16+
"org.mockito" %% "mockito-scala" % "1.16.42"
17+
).map(_ % Test)
18+
19+
val all: Seq[ModuleID] = prod ++ test
20+
}

‎project/SbtAliases.scala‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
object SbtAliases {
2+
val aliases: Seq[(String, String)] = Seq(
3+
"t" -> "test",
4+
"to" -> "testOnly",
5+
"tq" -> "testQuick",
6+
"tsf" -> "testShowFailed",
7+
"c" -> "compile",
8+
"tc" -> "Test / compile",
9+
"f" -> "scalafmt", // Format production files according to ScalaFmt
10+
"fc" -> "scalafmtCheck", // Check if production files are formatted according to ScalaFmt
11+
"tf" -> "Test / scalafmt", // Format test files according to ScalaFmt
12+
"tfc" -> "Test / scalafmtCheck", // Check if test files are formatted according to ScalaFmt
13+
"prep" -> ";c;tc;fc;tfc" // All the needed tasks before pushing to the repository (compile, compile test, format check in prod and test)
14+
)
15+
}

0 commit comments

Comments
 (0)