Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
Kile NIKLAWSKI
TwitterStreaming
Commits
1445abbb
Commit
1445abbb
authored
Mar 03, 2017
by
Jeannine Tondreau
Browse files
initial commit of code
parent
623c8f24
Changes
6
Hide whitespace changes
Inline
Side-by-side
.gitignore
0 → 100644
View file @
1445abbb
target
.project
.classpath
README.md
View file @
1445abbb
Twitter example for Spark Streaming
\ No newline at end of file
dependency-reduced-pom.xml
0 → 100644
View file @
1445abbb
<?xml version="1.0" encoding="UTF-8"?>
<project
xmlns=
"http://maven.apache.org/POM/4.0.0"
xmlns:xsi=
"http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation=
"http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"
>
<modelVersion>
4.0.0
</modelVersion>
<groupId>
com.ippon.dojo
</groupId>
<artifactId>
TwitterStreaming
</artifactId>
<version>
0.0.1-SNAPSHOT
</version>
<build>
<sourceDirectory>
src/main/scala
</sourceDirectory>
<plugins>
<plugin>
<artifactId>
maven-shade-plugin
</artifactId>
<version>
3.0.0
</version>
<executions>
<execution>
<phase>
package
</phase>
<goals>
<goal>
shade
</goal>
</goals>
<configuration>
<filters>
<filter>
<artifact>
*:*
</artifact>
<excludes>
<exclude>
META-INF/*.SF
</exclude>
<exclude>
META-INF/*.DSA
</exclude>
<exclude>
META-INF/*.RSA
</exclude>
</excludes>
</filter>
</filters>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<groupId>
net.alchim31.maven
</groupId>
<artifactId>
scala-maven-plugin
</artifactId>
<version>
3.2.1
</version>
<executions>
<execution>
<goals>
<goal>
compile
</goal>
<goal>
testCompile
</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>
pom.xml
0 → 100644
View file @
1445abbb
<project
xmlns=
"http://maven.apache.org/POM/4.0.0"
xmlns:xsi=
"http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation=
"http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"
>
<modelVersion>
4.0.0
</modelVersion>
<groupId>
com.ippon.dojo
</groupId>
<artifactId>
TwitterStreaming
</artifactId>
<version>
0.0.1-SNAPSHOT
</version>
<dependencies>
<dependency>
<groupId>
org.apache.spark
</groupId>
<artifactId>
spark-streaming_2.11
</artifactId>
<version>
2.1.0
</version>
</dependency>
<dependency>
<groupId>
org.apache.bahir
</groupId>
<artifactId>
spark-streaming-twitter_2.11
</artifactId>
<version>
2.1.0
</version>
</dependency>
</dependencies>
<build>
<sourceDirectory>
src/main/scala
</sourceDirectory>
<plugins>
<plugin>
<groupId>
org.apache.maven.plugins
</groupId>
<artifactId>
maven-shade-plugin
</artifactId>
<version>
3.0.0
</version>
<executions>
<execution>
<phase>
package
</phase>
<goals>
<goal>
shade
</goal>
</goals>
<configuration>
<filters>
<filter>
<artifact>
*:*
</artifact>
<excludes>
<exclude>
META-INF/*.SF
</exclude>
<exclude>
META-INF/*.DSA
</exclude>
<exclude>
META-INF/*.RSA
</exclude>
</excludes>
</filter>
</filters>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<groupId>
net.alchim31.maven
</groupId>
<artifactId>
scala-maven-plugin
</artifactId>
<version>
3.2.1
</version>
<executions>
<execution>
<goals>
<goal>
compile
</goal>
<goal>
testCompile
</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>
src/main/resources/twitter4j.properties
0 → 100644
View file @
1445abbb
oauth.consumerKey
=
****
oauth.consumerSecret
=
****
oauth.accessToken
=
****
oauth.accessTokenSecret
=
****
\ No newline at end of file
src/main/scala/com/ippon/dojo/driver/TwitterStreamingDriver.scala
0 → 100644
View file @
1445abbb
package
com.ippon.dojo.driver
import
org.apache.spark.SparkConf
import
org.apache.spark.SparkContext
import
org.apache.spark.streaming.StreamingContext
import
org.apache.spark.streaming.Seconds
import
org.apache.spark.streaming.twitter.TwitterUtils
object
TwitterStreamingDriver
{
def
main
(
args
:
Array
[
String
])
:
Unit
=
{
//configure the Streaming Context
val
sparkConf
=
new
SparkConf
().
setAppName
(
"TwitterStreamingDojoApp"
)
val
sparkContext
=
new
SparkContext
(
sparkConf
)
val
streamingContext
=
new
StreamingContext
(
sparkContext
,
Seconds
(
1
))
//create the stream
val
tweets
=
TwitterUtils
.
createStream
(
streamingContext
,
None
)
//perform action on the stream
tweets
.
foreachRDD
((
rdd
,
time
)
=>
{
val
count
=
rdd
.
count
()
System
.
out
.
println
(
count
+
" tweets were collected at "
+
time
)
})
//start the stream
streamingContext
.
start
()
streamingContext
.
awaitTermination
()
}
}
\ No newline at end of file
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment