Flink bootstrap
This commit is contained in:
commit
e763a7611b
35
.gitignore
vendored
Normal file
35
.gitignore
vendored
Normal file
|
|
@ -0,0 +1,35 @@
|
||||||
|
# Compiled class file
|
||||||
|
*.class
|
||||||
|
|
||||||
|
# Log file
|
||||||
|
*.log
|
||||||
|
|
||||||
|
# BlueJ files
|
||||||
|
*.ctxt
|
||||||
|
|
||||||
|
# Mobile Tools for Java (J2ME)
|
||||||
|
.mtj.tmp/
|
||||||
|
|
||||||
|
# Package Files #
|
||||||
|
*.jar
|
||||||
|
*.war
|
||||||
|
*.nar
|
||||||
|
*.ear
|
||||||
|
*.zip
|
||||||
|
*.tar.gz
|
||||||
|
*.rar
|
||||||
|
|
||||||
|
# virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml
|
||||||
|
hs_err_pid*
|
||||||
|
|
||||||
|
# Maven
|
||||||
|
target/
|
||||||
|
pom.xml.tag
|
||||||
|
pom.xml.releaseBackup
|
||||||
|
pom.xml.versionsBackup
|
||||||
|
pom.xml.next
|
||||||
|
release.properties
|
||||||
|
dependency-reduced-pom.xml
|
||||||
|
buildNumber.properties
|
||||||
|
.mvn/timing.properties
|
||||||
|
.mvn/wrapper/maven-wrapper.jar
|
||||||
232
pom.xml
Normal file
232
pom.xml
Normal file
|
|
@ -0,0 +1,232 @@
|
||||||
|
<!--
|
||||||
|
Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
or more contributor license agreements. See the NOTICE file
|
||||||
|
distributed with this work for additional information
|
||||||
|
regarding copyright ownership. The ASF licenses this file
|
||||||
|
to you under the Apache License, Version 2.0 (the
|
||||||
|
"License"); you may not use this file except in compliance
|
||||||
|
with the License. You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing,
|
||||||
|
software distributed under the License is distributed on an
|
||||||
|
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||||
|
KIND, either express or implied. See the License for the
|
||||||
|
specific language governing permissions and limitations
|
||||||
|
under the License.
|
||||||
|
-->
|
||||||
|
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||||
|
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||||
|
<modelVersion>4.0.0</modelVersion>
|
||||||
|
|
||||||
|
<groupId>it.polimi.middleware.projects.flink</groupId>
|
||||||
|
<artifactId>project</artifactId>
|
||||||
|
<version>1.0-SNAPSHOT</version>
|
||||||
|
<packaging>jar</packaging>
|
||||||
|
|
||||||
|
<name>Flink Quickstart Job</name>
|
||||||
|
<url>http://www.myorganization.org</url>
|
||||||
|
|
||||||
|
<properties>
|
||||||
|
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||||
|
<flink.version>1.7.0</flink.version>
|
||||||
|
<java.version>1.8</java.version>
|
||||||
|
<scala.binary.version>2.11</scala.binary.version>
|
||||||
|
<maven.compiler.source>${java.version}</maven.compiler.source>
|
||||||
|
<maven.compiler.target>${java.version}</maven.compiler.target>
|
||||||
|
</properties>
|
||||||
|
|
||||||
|
<repositories>
|
||||||
|
<repository>
|
||||||
|
<id>apache.snapshots</id>
|
||||||
|
<name>Apache Development Snapshot Repository</name>
|
||||||
|
<url>https://repository.apache.org/content/repositories/snapshots/</url>
|
||||||
|
<releases>
|
||||||
|
<enabled>false</enabled>
|
||||||
|
</releases>
|
||||||
|
<snapshots>
|
||||||
|
<enabled>true</enabled>
|
||||||
|
</snapshots>
|
||||||
|
</repository>
|
||||||
|
</repositories>
|
||||||
|
|
||||||
|
<dependencies>
|
||||||
|
<!-- Apache Flink dependencies -->
|
||||||
|
<!-- These dependencies are provided, because they should not be packaged into the JAR file. -->
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.flink</groupId>
|
||||||
|
<artifactId>flink-java</artifactId>
|
||||||
|
<version>${flink.version}</version>
|
||||||
|
<scope>provided</scope>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.flink</groupId>
|
||||||
|
<artifactId>flink-streaming-java_${scala.binary.version}</artifactId>
|
||||||
|
<version>${flink.version}</version>
|
||||||
|
<scope>provided</scope>
|
||||||
|
</dependency>
|
||||||
|
|
||||||
|
<!-- Add connector dependencies here. They must be in the default scope (compile). -->
|
||||||
|
|
||||||
|
<!-- Example:
|
||||||
|
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.flink</groupId>
|
||||||
|
<artifactId>flink-connector-kafka-0.10_${scala.binary.version}</artifactId>
|
||||||
|
<version>${flink.version}</version>
|
||||||
|
</dependency>
|
||||||
|
-->
|
||||||
|
|
||||||
|
<!-- Add logging framework, to produce console output when running in the IDE. -->
|
||||||
|
<!-- These dependencies are excluded from the application JAR by default. -->
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.slf4j</groupId>
|
||||||
|
<artifactId>slf4j-log4j12</artifactId>
|
||||||
|
<version>1.7.7</version>
|
||||||
|
<scope>runtime</scope>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>log4j</groupId>
|
||||||
|
<artifactId>log4j</artifactId>
|
||||||
|
<version>1.2.17</version>
|
||||||
|
<scope>runtime</scope>
|
||||||
|
</dependency>
|
||||||
|
</dependencies>
|
||||||
|
|
||||||
|
<build>
|
||||||
|
<plugins>
|
||||||
|
|
||||||
|
<!-- Java Compiler -->
|
||||||
|
<plugin>
|
||||||
|
<groupId>org.apache.maven.plugins</groupId>
|
||||||
|
<artifactId>maven-compiler-plugin</artifactId>
|
||||||
|
<version>3.1</version>
|
||||||
|
<configuration>
|
||||||
|
<source>${java.version}</source>
|
||||||
|
<target>${java.version}</target>
|
||||||
|
</configuration>
|
||||||
|
</plugin>
|
||||||
|
|
||||||
|
<!-- We use the maven-shade plugin to create a fat jar that contains all necessary dependencies. -->
|
||||||
|
<!-- Change the value of <mainClass>...</mainClass> if your program entry point changes. -->
|
||||||
|
<plugin>
|
||||||
|
<groupId>org.apache.maven.plugins</groupId>
|
||||||
|
<artifactId>maven-shade-plugin</artifactId>
|
||||||
|
<version>3.0.0</version>
|
||||||
|
<executions>
|
||||||
|
<!-- Run shade goal on package phase -->
|
||||||
|
<execution>
|
||||||
|
<phase>package</phase>
|
||||||
|
<goals>
|
||||||
|
<goal>shade</goal>
|
||||||
|
</goals>
|
||||||
|
<configuration>
|
||||||
|
<artifactSet>
|
||||||
|
<excludes>
|
||||||
|
<exclude>org.apache.flink:force-shading</exclude>
|
||||||
|
<exclude>com.google.code.findbugs:jsr305</exclude>
|
||||||
|
<exclude>org.slf4j:*</exclude>
|
||||||
|
<exclude>log4j:*</exclude>
|
||||||
|
</excludes>
|
||||||
|
</artifactSet>
|
||||||
|
<filters>
|
||||||
|
<filter>
|
||||||
|
<!-- Do not copy the signatures in the META-INF folder.
|
||||||
|
Otherwise, this might cause SecurityExceptions when using the JAR. -->
|
||||||
|
<artifact>*:*</artifact>
|
||||||
|
<excludes>
|
||||||
|
<exclude>META-INF/*.SF</exclude>
|
||||||
|
<exclude>META-INF/*.DSA</exclude>
|
||||||
|
<exclude>META-INF/*.RSA</exclude>
|
||||||
|
</excludes>
|
||||||
|
</filter>
|
||||||
|
</filters>
|
||||||
|
<transformers>
|
||||||
|
<transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
|
||||||
|
<mainClass>it.polimi.middleware.projects.flink.StreamingJob</mainClass>
|
||||||
|
</transformer>
|
||||||
|
</transformers>
|
||||||
|
</configuration>
|
||||||
|
</execution>
|
||||||
|
</executions>
|
||||||
|
</plugin>
|
||||||
|
</plugins>
|
||||||
|
|
||||||
|
<pluginManagement>
|
||||||
|
<plugins>
|
||||||
|
|
||||||
|
<!-- This improves the out-of-the-box experience in Eclipse by resolving some warnings. -->
|
||||||
|
<plugin>
|
||||||
|
<groupId>org.eclipse.m2e</groupId>
|
||||||
|
<artifactId>lifecycle-mapping</artifactId>
|
||||||
|
<version>1.0.0</version>
|
||||||
|
<configuration>
|
||||||
|
<lifecycleMappingMetadata>
|
||||||
|
<pluginExecutions>
|
||||||
|
<pluginExecution>
|
||||||
|
<pluginExecutionFilter>
|
||||||
|
<groupId>org.apache.maven.plugins</groupId>
|
||||||
|
<artifactId>maven-shade-plugin</artifactId>
|
||||||
|
<versionRange>[3.0.0,)</versionRange>
|
||||||
|
<goals>
|
||||||
|
<goal>shade</goal>
|
||||||
|
</goals>
|
||||||
|
</pluginExecutionFilter>
|
||||||
|
<action>
|
||||||
|
<ignore/>
|
||||||
|
</action>
|
||||||
|
</pluginExecution>
|
||||||
|
<pluginExecution>
|
||||||
|
<pluginExecutionFilter>
|
||||||
|
<groupId>org.apache.maven.plugins</groupId>
|
||||||
|
<artifactId>maven-compiler-plugin</artifactId>
|
||||||
|
<versionRange>[3.1,)</versionRange>
|
||||||
|
<goals>
|
||||||
|
<goal>testCompile</goal>
|
||||||
|
<goal>compile</goal>
|
||||||
|
</goals>
|
||||||
|
</pluginExecutionFilter>
|
||||||
|
<action>
|
||||||
|
<ignore/>
|
||||||
|
</action>
|
||||||
|
</pluginExecution>
|
||||||
|
</pluginExecutions>
|
||||||
|
</lifecycleMappingMetadata>
|
||||||
|
</configuration>
|
||||||
|
</plugin>
|
||||||
|
</plugins>
|
||||||
|
</pluginManagement>
|
||||||
|
</build>
|
||||||
|
|
||||||
|
<!-- This profile helps to make things run out of the box in IntelliJ -->
|
||||||
|
<!-- Its adds Flink's core classes to the runtime class path. -->
|
||||||
|
<!-- Otherwise they are missing in IntelliJ, because the dependency is 'provided' -->
|
||||||
|
<profiles>
|
||||||
|
<profile>
|
||||||
|
<id>add-dependencies-for-IDEA</id>
|
||||||
|
|
||||||
|
<activation>
|
||||||
|
<property>
|
||||||
|
<name>idea.version</name>
|
||||||
|
</property>
|
||||||
|
</activation>
|
||||||
|
|
||||||
|
<dependencies>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.flink</groupId>
|
||||||
|
<artifactId>flink-java</artifactId>
|
||||||
|
<version>${flink.version}</version>
|
||||||
|
<scope>compile</scope>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.flink</groupId>
|
||||||
|
<artifactId>flink-streaming-java_${scala.binary.version}</artifactId>
|
||||||
|
<version>${flink.version}</version>
|
||||||
|
<scope>compile</scope>
|
||||||
|
</dependency>
|
||||||
|
</dependencies>
|
||||||
|
</profile>
|
||||||
|
</profiles>
|
||||||
|
|
||||||
|
</project>
|
||||||
|
|
@ -0,0 +1,66 @@
|
||||||
|
/*
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package it.polimi.middleware.projects.flink;
|
||||||
|
|
||||||
|
import org.apache.flink.api.java.ExecutionEnvironment;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Skeleton for a Flink Batch Job.
|
||||||
|
*
|
||||||
|
* <p>For a tutorial how to write a Flink batch application, check the
|
||||||
|
* tutorials and examples on the <a href="http://flink.apache.org/docs/stable/">Flink Website</a>.
|
||||||
|
*
|
||||||
|
* <p>To package your application into a JAR file for execution,
|
||||||
|
* change the main class in the POM.xml file to this class (simply search for 'mainClass')
|
||||||
|
* and run 'mvn clean package' on the command line.
|
||||||
|
*/
|
||||||
|
public class BatchJob {
|
||||||
|
|
||||||
|
public static void main(String[] args) throws Exception {
|
||||||
|
// set up the batch execution environment
|
||||||
|
final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Here, you can start creating your execution plan for Flink.
|
||||||
|
*
|
||||||
|
* Start with getting some data from the environment, like
|
||||||
|
* env.readTextFile(textPath);
|
||||||
|
*
|
||||||
|
* then, transform the resulting DataSet<String> using operations
|
||||||
|
* like
|
||||||
|
* .filter()
|
||||||
|
* .flatMap()
|
||||||
|
* .join()
|
||||||
|
* .coGroup()
|
||||||
|
*
|
||||||
|
* and many more.
|
||||||
|
* Have a look at the programming guide for the Java API:
|
||||||
|
*
|
||||||
|
* http://flink.apache.org/docs/latest/apis/batch/index.html
|
||||||
|
*
|
||||||
|
* and the examples
|
||||||
|
*
|
||||||
|
* http://flink.apache.org/docs/latest/apis/batch/examples.html
|
||||||
|
*
|
||||||
|
*/
|
||||||
|
|
||||||
|
// execute program
|
||||||
|
env.execute("Flink Batch Java API Skeleton");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,64 @@
|
||||||
|
/*
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package it.polimi.middleware.projects.flink;
|
||||||
|
|
||||||
|
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Skeleton for a Flink Streaming Job.
|
||||||
|
*
|
||||||
|
* <p>For a tutorial how to write a Flink streaming application, check the
|
||||||
|
* tutorials and examples on the <a href="http://flink.apache.org/docs/stable/">Flink Website</a>.
|
||||||
|
*
|
||||||
|
* <p>To package your application into a JAR file for execution, run
|
||||||
|
* 'mvn clean package' on the command line.
|
||||||
|
*
|
||||||
|
* <p>If you change the name of the main class (with the public static void main(String[] args))
|
||||||
|
* method, change the respective entry in the POM.xml file (simply search for 'mainClass').
|
||||||
|
*/
|
||||||
|
public class StreamingJob {
|
||||||
|
|
||||||
|
public static void main(String[] args) throws Exception {
|
||||||
|
// set up the streaming execution environment
|
||||||
|
final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Here, you can start creating your execution plan for Flink.
|
||||||
|
*
|
||||||
|
* Start with getting some data from the environment, like
|
||||||
|
* env.readTextFile(textPath);
|
||||||
|
*
|
||||||
|
* then, transform the resulting DataStream<String> using operations
|
||||||
|
* like
|
||||||
|
* .filter()
|
||||||
|
* .flatMap()
|
||||||
|
* .join()
|
||||||
|
* .coGroup()
|
||||||
|
*
|
||||||
|
* and many more.
|
||||||
|
* Have a look at the programming guide for the Java API:
|
||||||
|
*
|
||||||
|
* http://flink.apache.org/docs/latest/apis/streaming/index.html
|
||||||
|
*
|
||||||
|
*/
|
||||||
|
|
||||||
|
// execute program
|
||||||
|
env.execute("Flink Streaming Java API Skeleton");
|
||||||
|
}
|
||||||
|
}
|
||||||
23
src/main/resources/log4j.properties
Normal file
23
src/main/resources/log4j.properties
Normal file
|
|
@ -0,0 +1,23 @@
|
||||||
|
################################################################################
|
||||||
|
# Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
# or more contributor license agreements. See the NOTICE file
|
||||||
|
# distributed with this work for additional information
|
||||||
|
# regarding copyright ownership. The ASF licenses this file
|
||||||
|
# to you under the Apache License, Version 2.0 (the
|
||||||
|
# "License"); you may not use this file except in compliance
|
||||||
|
# with the License. You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
################################################################################
|
||||||
|
|
||||||
|
log4j.rootLogger=INFO, console
|
||||||
|
|
||||||
|
log4j.appender.console=org.apache.log4j.ConsoleAppender
|
||||||
|
log4j.appender.console.layout=org.apache.log4j.PatternLayout
|
||||||
|
log4j.appender.console.layout.ConversionPattern=%d{HH:mm:ss,SSS} %-5p %-60c %x - %m%n
|
||||||
Reference in a new issue